{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "\n",
    "dpath = './data/'\n",
    "dtrain = pd.read_csv(dpath + 'RentListingInquries_FE_train.csv')\n",
    "dtest = pd.read_csv(dpath + 'RentListingInquries_FE_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "      <th>interest_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.5</td>\n",
       "      <td>3</td>\n",
       "      <td>3000</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>750.000000</td>\n",
       "      <td>-1.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>5465</td>\n",
       "      <td>2732.5</td>\n",
       "      <td>1821.666667</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.0</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>17</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3275</td>\n",
       "      <td>1637.5</td>\n",
       "      <td>1637.500000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>18</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>4</td>\n",
       "      <td>3350</td>\n",
       "      <td>1675.0</td>\n",
       "      <td>670.000000</td>\n",
       "      <td>-3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 228 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.5         3   3000           1200.0      750.000000       -1.5   \n",
       "1        1.0         2   5465           2732.5     1821.666667       -1.0   \n",
       "2        1.0         1   2850           1425.0     1425.000000        0.0   \n",
       "3        1.0         1   3275           1637.5     1637.500000        0.0   \n",
       "4        1.0         4   3350           1675.0      670.000000       -3.0   \n",
       "\n",
       "   room_num  Year  Month  Day       ...        walk  walls  war  washer  \\\n",
       "0       4.5  2016      6   24       ...           0      0    0       0   \n",
       "1       3.0  2016      6   12       ...           0      0    0       0   \n",
       "2       2.0  2016      4   17       ...           0      0    0       0   \n",
       "3       2.0  2016      4   18       ...           0      0    0       0   \n",
       "4       5.0  2016      4   28       ...           0      0    1       0   \n",
       "\n",
       "   water  wheelchair  wifi  windows  work  interest_level  \n",
       "0      0           0     0        0     0               1  \n",
       "1      0           0     0        0     0               2  \n",
       "2      0           0     0        0     0               0  \n",
       "3      0           0     0        0     0               2  \n",
       "4      0           0     0        0     0               2  \n",
       "\n",
       "[5 rows x 228 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtrain.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAELCAYAAAARNxsIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGx5JREFUeJzt3X2wHXWd5/H3h/AgPibIhcokYYKaGUUdo14hypSj6EDAHRMpUChHIkNtxAKF0rUEd5aMPMyM4ygrM8pupogkrkOIiBI1GDMMyII8JGB4CJHlioxcyZJgeNQVivDZP/p35eTm5N7OpU9OTu7nVdV1ur/96z7fwy340v379a9lm4iIiCbs0e0EIiJi95GiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIas2e3E9jZ9t9/f0+fPr3baURE9JTbbrvtEdt9o7Ubd0Vl+vTprFmzpttpRET0FEn/Uaddbn9FRERjOlZUJL1I0q2S7pC0TtLnS/xSSb+QtLYsM0tcki6SNCDpTklvaTnXPEn3lWVeS/ytku4qx1wkSZ36PRERMbpO3v56GjjC9lOS9gJukHR12fcZ21cMa380MKMshwEXA4dJ2g9YAPQDBm6TtNz2o6XNfOBmYAUwG7iaiIjoio5dqbjyVNncqywjzbM/B1hSjrsZmChpMnAUsMr25lJIVgGzy76X277J1fz9S4C5nfo9ERExuo72qUiaIGktsJGqMNxSdl1QbnFdKGmfEpsCPNhy+GCJjRQfbBNvl8d8SWskrdm0adML/l0REdFeR4uK7S22ZwJTgUMlvQE4G3gt8DZgP+CzpXm7/hCPId4uj4W2+2339/WNOiIuIiLGaKeM/rL9GHAdMNv2hnKL62ng68ChpdkgMK3lsKnAQ6PEp7aJR0REl3Ry9FefpIllfV/gvcDPSl8IZaTWXODucshy4KQyCmwW8LjtDcBK4EhJkyRNAo4EVpZ9T0qaVc51EnBVp35PRESMrpOjvyYDiyVNoCpey2x/X9K/S+qjun21Fji1tF8BHAMMAL8FTgawvVnSecDq0u5c25vL+seBS4F9qUZ9ZeRXREQXqRo4NX709/c7T9RH7NoO/6fDu53Cbu/GT9y4Q+0l3Wa7f7R2eaI+IiIak6ISERGNSVGJiIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYmIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMR0rKpJeJOlWSXdIWifp8yV+sKRbJN0n6XJJe5f4PmV7oOyf3nKus0v8XklHtcRnl9iApLM69VsiIqKeTl6pPA0cYftNwExgtqRZwBeAC23PAB4FTintTwEetf0a4MLSDkmHACcArwdmA1+TNEHSBOCrwNHAIcCJpW1ERHRJx4qKK0+Vzb3KYuAI4IoSXwzMLetzyjZl/3skqcSX2n7a9i+AAeDQsgzYvt/2M8DS0jYiIrqko30q5YpiLbARWAX8HHjM9rOlySAwpaxPAR4EKPsfB17ZGh92zPbiERHRJR0tKra32J4JTKW6snhdu2blU9vZt6PxbUiaL2mNpDWbNm0aPfGIiBiTnTL6y/ZjwHXALGCipD3LrqnAQ2V9EJgGUPa/AtjcGh92zPbi7b5/oe1+2/19fX1N/KSIiGijk6O/+iRNLOv7Au8F1gPXAseVZvOAq8r68rJN2f/vtl3iJ5TRYQcDM4BbgdXAjDKabG+qzvzlnfo9ERExuj1HbzJmk4HFZZTWHsAy29+XdA+wVNL5wE+BS0r7S4BvSBqgukI5AcD2OknLgHuAZ4HTbG8BkHQ6sBKYACyyva6DvyciIkbRsaJi+07gzW3i91P1rwyP/w44fjvnugC4oE18BbDiBScbERGNyBP1ERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGpKhERERjUlQiIqIxKSoREdGYFJWIiGjMqEVF0ksk7VHW/0jS+yXt1fnUIiKi19S5UrkeeJGkKcA1wMnApZ1MKiIielOdoiLbvwWOBf7J9geo3gkfERGxlVpFRdLbgQ8DPyixTk6ZHxERPapOUTkTOBv4Tnm3yauoXrQVERGxlVGvOGz/GPixpJeU7fuBT3Y6sYiI6D11Rn+9vbytcX3ZfpOkr3U8s4iI6Dl1bn/9d+Ao4NcAtu8A3tnJpCIiojfVevjR9oPDQls6kEtERPS4OqO4HpT0DsCS9qbqT1nf2bQiIqIX1blSORU4DZgCDAIzy3ZERMRWRi0qth+x/WHbB9o+wPZf2v71aMdJmibpWknrJa2TdEaJ/42kX0laW5ZjWo45W9KApHslHdUSn11iA5LOaokfLOkWSfdJurxcSUVERJfUGf21WNLElu1JkhbVOPezwKdtvw6YBZwmaehJ/AttzyzLinLeQ4ATgNcDs4GvSZogaQLwVeBoqif5T2w5zxfKuWYAjwKn1MgrIiI6pM7trz+x/djQhu1HgTePdpDtDbZvL+tPUvXDTBnhkDnAUttP2/4FMAAcWpYB2/fbfgZYCsyRJOAI4Ipy/GJgbo3fExERHVKnqOwhadLQhqT92MFpWiRNpypEt5TQ6ZLulLSo5dxTgNZRZoMltr34K4HHbD87LB4REV1Sp6h8CfiJpPMknQf8BPiHul8g6aXAt4EzbT8BXAy8mqrDf0M5P4DaHO4xxNvlMF/SGklrNm3aVDf1iIjYQXU66pcAxwEPAxuBY21/o87Jy3tXvg180/aV5XwP295i+zngX6hub0F1pTGt5fCpwEMjxB8BJkrac1i83W9YaLvfdn9fX1+d1CMiYgzqvvnxZ8CVwFXAU5IOGu2A0udxCbDe9pdb4pNbmn0AuLusLwdOkLSPpIOBGcCtwGpgRhnptTdVZ/5y26aa2PK4cvy8kl9ERHTJqH0jkj4BLKC6UtlCddvJwJ+McujhwEeAuyStLbHPUY3emlnO8QDwMYAyA/Iy4B6qkWOn2d5ScjgdWAlMABbZXlfO91lgqaTzgZ9SFbGIiOiSOh3uZwB/XOfZlFa2b6B9v8eKEY65ALigTXxFu+PKjMmHDo9HRER31Ln99SDweKcTiYiI3lfnSuV+4DpJPwCeHgq29pNERERAvaLyy7LsXZaIiIi26rz58fMAkl5i+zedTykiInpV3vwYERGNyZsfIyKiMXnzY0RENCZvfoyIiMbkzY8REdGYEa9UyguyPmL7wzspn4iI6GEjXqmUubfm7KRcIiKix9XpU7lR0j8DlwO/f05l6K2OERERQ+oUlXeUz3NbYqZ6lW9ERMTvjdansgdwse1lOymfiIjoYaP1qTwHnL6TcomIiB5XZ0jxKkn/RdI0SfsNLR3PLCIiek6dPpW/Kp+tz6YYeFXz6URERC+rM0vxwTsjkYiI6H113lF/Uru47SXNpxMREb2szu2vt7Wsvwh4D3A7kKISERFbqXP76xOt25JeAXyjYxlFRETPqjX1/TC/BWaM1qiMFrtW0npJ6ySdUeL7SVol6b7yOanEJekiSQOS7pT0lpZzzSvt75M0ryX+Vkl3lWMukqQx/J6IiGhInTc/fk/S8rJ8H7gXuKrGuZ8FPm37dcAs4DRJhwBnAdfYngFcU7YBjqYqVjOA+cDF5fv3AxYAhwGHAguGClFpM7/luNk18oqIiA6p06fyjy3rzwL/YXtwtINsbwA2lPUnJa2nmj5/DvCu0mwxcB3w2RJfYtvAzZImSppc2q6yvRlA0ipgtqTrgJfbvqnElwBzgatr/KaIiOiAOkXll8AG278DkLSvpOm2H6j7JZKmA28GbgEOLAUH2xskHVCaTQFa3zA5WGIjxQfbxCMiokvq9Kl8C3iuZXtLidUi6aXAt4EzbT8xUtM2MY8h3i6H+ZLWSFqzadOm0VKOiIgxqlNU9rT9zNBGWd+7zskl7UVVUL5p+8oSfrjc1qJ8bizxQWBay+FTgYdGiU9tE9+G7YW2+2339/X11Uk9IiLGoE5R2STp/UMbkuYAj4x2UBmJdQmw3vaXW3YtB4ZGcM3j+U7/5cBJZRTYLODxcptsJXCkpEmlg/5IYGXZ96SkWeW7TqLeAIKIiOiQOn0qpwLfLC/qguoKoe1T9sMcDnwEuEvS2hL7HPD3wDJJp1D11xxf9q0AjgEGqIYtnwxge7Ok84DVpd25Q532wMeBS4F9qTro00kfEdFFdR5+/Dkwq/SNyPaTdU5s+wba93tA9VT+8PZm60krW/ctAha1ia8B3lAnn4iI6Lw6z6n8raSJtp8qQ4MnSTp/ZyQXERG9pU6fytG2HxvasP0o1W2qiIiIrdQpKhMk7TO0IWlfYJ8R2kdExDhVp6P+fwHXSPo61XMgf0X1JHxERMRW6nTU/4OkO4H3ltB5tld2Nq2IiOhFda5UAH4K7EV1pfLTzqUTERG9rM7orw8CtwLHAR8EbpF0XKcTi4iI3lPnSuW/Am+zvRFAUh/wb8AVnUwsIiJ6T53RX3sMFZTi1zWPi4iIcabOlcoPJa0ELivbH6KaUiUiImIrdUZ/fUbSscCfUk27stD2dzqeWURE9Jxao7/KtPVXjtowIiLGtfSNREREY1JUIiKiMdstKpKuKZ9f2HnpRERELxupT2WypD8D3i9pKcPejWL79o5mFhERPWekonIOcBbVu9+/PGyfgSM6lVRERPSm7RYV21cAV0j6b7bP24k5RUREj6rznMp5kt4PvLOErrP9/c6mFRERvajOhJJ/B5wB3FOWM0osIiJiK3UefnwfMNP2cwCSFlNNf392JxOLiIjeU/c5lYkt66/oRCIREdH76hSVvwN+KunScpVyG/C3ox0kaZGkjZLubon9jaRfSVpblmNa9p0taUDSvZKOaonPLrEBSWe1xA+WdIuk+yRdLmnvuj86IiI6Y9SiYvsyYBbV3F9XAm+3vbTGuS8FZreJX2h7ZllWAEg6BDgBeH055muSJkiaAHwVOBo4BDixtAX4QjnXDOBR4JQaOUVERAfVuv1le4Pt5bavsv1/ax5zPbC5Zh5zgKW2n7b9C2AAOLQsA7bvt/0MsBSYI0lUz8kMvShsMTC35ndFRESHdGPur9Ml3Vluj00qsSnAgy1tBktse/FXAo/ZfnZYvC1J8yWtkbRm06ZNTf2OiIgYZmcXlYuBVwMzgQ3Al0pcbdp6DPG2bC+03W+7v6+vb8cyjoiI2kYsKpL2aO1of6FsP2x7Sxme/C9Ut7egutKY1tJ0KvDQCPFHgImS9hwWj4iILhqxqJT/+N8h6aAmvkzS5JbNDwBDBWs5cIKkfSQdDMwAbgVWAzPKSK+9qTrzl9s2cC1wXDl+HnBVEzlGRMTY1Xn4cTKwTtKtwG+GgrbfP9JBki4D3gXsL2kQWAC8S9JMqltVDwAfK+daJ2kZ1RP7zwKn2d5SznM6sBKYACyyva58xWeBpZLOp3oY85I6PzgiIjqnTlH5/FhObPvENuHt/off9gXABW3iK4AVbeL38/zts4iI2AXUmVDyx5L+EJhh+98kvZjqqiEiImIrdSaU/M9Uz4P8zxKaAny3k0lFRERvqjOk+DTgcOAJANv3AQd0MqmIiOhNdYrK0+VpdgDKMN7tPhMSERHjV52i8mNJnwP2lfTnwLeA73U2rYiI6EV1ispZwCbgLqohwCuAv+5kUhER0ZvqjP56rkx5fwvVba97y8OHERERWxm1qEh6H/A/gJ9Tzbl1sKSP2b6608lFRERvqfPw45eAd9seAJD0auAHQIpKRERspU6fysahglLcD2zsUD4REdHDtnulIunYsrpO0gpgGVWfyvFUEz1GRERsZaTbX3/Rsv4w8GdlfRMwadvmEREx3m23qNg+eWcmEhERva/O6K+DgU8A01vbjzb1fUREjD91Rn99l2rK+u8Bz3U2nYiI6GV1isrvbF/U8UwiIqLn1SkqX5G0APgR8PRQ0PbtHcsqIiJ6Up2i8kbgI8ARPH/7y2U7Ypf0y3Pf2O0UxoWDzrmr2ynELqZOUfkA8KrW6e8jIiLaqfNE/R3AxE4nEhERva/OlcqBwM8krWbrPpUMKY6IiK3UKSoLxnJiSYuA/0Q1d9gbSmw/4HKqZ14eAD5o+1FJAr4CHAP8Fvjo0EAASfN4/v0t59teXOJvBS4F9qV6x8sZmZI/IqK7Rr39ZfvH7ZYa574UmD0sdhZwje0ZwDVlG+BoYEZZ5gMXw++L0ALgMOBQYIGkoSliLi5th44b/l0REbGTjVpUJD0p6Ymy/E7SFklPjHac7euBzcPCc4DFZX0xMLclvsSVm4GJkiYDRwGrbG+2/SiwCphd9r3c9k3l6mRJy7kiIqJL6rz58WWt25LmUl01jMWBtjeU826QdECJTwEebGk3WGIjxQfbxNuSNJ/qqoaDDjpojKlHRMRo6oz+2ort79L8Mypq91VjiLdle6Htftv9fX19Y0wxIiJGU2dCyWNbNvcA+hnhP+CjeFjS5HKVMpnnX/Y1CExraTcVeKjE3zUsfl2JT23TPiIiuqjOlcpftCxHAU9S9YGMxXJgXlmfB1zVEj9JlVnA4+U22UrgSEmTSgf9kcDKsu9JSbPKyLGTWs4VERFdUqdPZUzvVZF0GdVVxv6SBqlGcf09sEzSKcAvqd4iCdWQ4GOAAaohxSeX794s6Tyef9PkubaHOv8/zvNDiq8uS0REdNFIrxM+Z4TjbPu8kU5s+8Tt7HpPu5MBp23nPIuARW3ia4A3jJRDRETsXCNdqfymTewlwCnAK4ERi0pERIw/I71O+EtD65JeBpxBdVtqKfCl7R0XERHj14h9KuWJ9k8BH6Z6WPEt5SHEiIiIbYzUp/JF4FhgIfBG20/ttKwiIqInjTSk+NPAH1BN5vhQy1QtT9aZpiUiIsafkfpUdvhp+4iIGN9SOCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGdKWoSHpA0l2S1kpaU2L7SVol6b7yOanEJekiSQOS7pT0lpbzzCvt75M0rxu/JSIintfNK5V3255pu79snwVcY3sGcE3ZBjgamFGW+cDFUBUhYAFwGHAosGCoEEVERHfsSre/5gCLy/piYG5LfIkrNwMTJU0GjgJW2d5s+1FgFTB7ZycdERHP61ZRMfAjSbdJml9iB9reAFA+DyjxKcCDLccOltj24hER0SXbfUd9hx1u+yFJBwCrJP1shLZqE/MI8W1PUBWu+QAHHXTQjuYaERE1deVKxfZD5XMj8B2qPpGHy20tyufG0nwQmNZy+FTgoRHi7b5voe1+2/19fX1N/pSIiGix04uKpJdIetnQOnAkcDewHBgawTUPuKqsLwdOKqPAZgGPl9tjK4EjJU0qHfRHllhERHRJN25/HQh8R9LQ9/+r7R9KWg0sk3QK8Evg+NJ+BXAMMAD8FjgZwPZmSecBq0u7c21v3nk/IyIihtvpRcX2/cCb2sR/DbynTdzAads51yJgUdM5RkTE2OxKQ4ojIqLHpahERERjujWkuCe89TNLup3Cbu+2L57U7RQiokG5UomIiMakqERERGNSVCIiojEpKhER0ZgUlYiIaEyKSkRENCZFJSIiGpOiEhERjUlRiYiIxqSoREREY1JUIiKiMSkqERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGNSVGJiIjGpKhERERjer6oSJot6V5JA5LO6nY+ERHjWU8XFUkTgK8CRwOHACdKOqS7WUVEjF89XVSAQ4EB2/fbfgZYCszpck4REeNWrxeVKcCDLduDJRYREV2wZ7cTeIHUJuZtGknzgfll8ylJ93Y0q+7aH3ik20nUpX+c1+0UdiU99bcDYEG7fwXHrZ76++mTO/y3+8M6jXq9qAwC01q2pwIPDW9keyGwcGcl1U2S1tju73YesePyt+tt+ftVev3212pghqSDJe0NnAAs73JOERHjVk9fqdh+VtLpwEpgArDI9roupxURMW71dFEBsL0CWNHtPHYh4+I2324qf7velr8fIHubfu2IiIgx6fU+lYiI2IWkqOwmMl1N75K0SNJGSXd3O5fYMZKmSbpW0npJ6ySd0e2cui23v3YDZbqa/wP8OdUw69XAibbv6WpiUYukdwJPAUtsv6Hb+UR9kiYDk23fLullwG3A3PH8716uVHYPma6mh9m+Htjc7Txix9neYPv2sv4ksJ5xPqtHisruIdPVRHSZpOnAm4FbuptJd6Wo7B5qTVcTEZ0h6aXAt4EzbT/R7Xy6KUVl91BrupqIaJ6kvagKyjdtX9ntfLotRWX3kOlqIrpAkoBLgPW2v9ztfHYFKSq7AdvPAkPT1awHlmW6mt4h6TLgJuCPJQ1KOqXbOUVthwMfAY6QtLYsx3Q7qW7KkOKIiGhMrlQiIqIxKSoREdGYFJWIiGhMikpERDQmRSUiIhqTohIREY1JUYkAJP2kRpszJb24w3nMHO05B0kflfTPDX9v4+eM8SlFJQKw/Y4azc4EdqiolNcS7IiZwLh+eC56W4pKBCDpqfL5LknXSbpC0s8kfVOVTwJ/AFwr6drS9khJN0m6XdK3yqSCSHpA0jmSbgCOl/RqST+UdJuk/y3ptaXd8ZLulnSHpOvLFDvnAh8qT2Z/qEbefZK+LWl1WQ6XtEfJYWJLuwFJB7Zr3/g/zBjX9ux2AhG7oDcDr6ealPNG4HDbF0n6FPBu249I2h/4a+C9tn8j6bPAp6iKAsDvbP8pgKRrgFNt3yfpMOBrwBHAOcBRtn8laaLtZySdA/TbPr1mrl8BLrR9g6SDgJW2XyfpKuADwNfLdz5g+2FJ/zq8PfC6F/jPK+L3UlQitnWr7UEASWuB6cANw9rMAg4BbqzmFGRvqvm7hlxejn8p8A7gW6UdwD7l80bgUknLgLHObvte4JCWc7+8vIHwcqqi9XWqCUYvH6V9RCNSVCK29XTL+hba/3siYJXtE7dzjt+Uzz2Ax2zPHN7A9qnlKuJ9wFpJ27SpYQ/g7bb/31bJSTcBr5HUB8wFzh+l/Ri+OmJb6VOJqO9JYOj/6m8GDpf0GgBJL5b0R8MPKC9s+oWk40s7SXpTWX+17VtsnwM8QvVOnNbvqONHVDNUU845s3yvge8AX6aalv3XI7WPaEqKSkR9C4GrJV1rexPwUeAySXdSFZnXbue4DwOnSLoDWAfMKfEvSrpL0t3A9cAdwLVUt6dqddQDnwT6Jd0p6R7g1JZ9lwN/yfO3vkZrH/GCZer7iIhoTK5UIiKiMemoj9hFSToZOGNY+Ebbp3Ujn4g6cvsrIiIak9tfERHRmBSViIhoTIpKREQ0JkUlIiIak6ISERGN+f+UQGoL+CmGNwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7d442e6e50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(dtrain.interest_level);\n",
    "pyplot.xlabel('interest_level');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "样本分类不均，使用交叉验证时按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 49352 entries, 0 to 49351\n",
      "Columns: 228 entries, bathrooms to interest_level\n",
      "dtypes: float64(9), int64(219)\n",
      "memory usage: 85.8 MB\n"
     ]
    }
   ],
   "source": [
    "dtrain.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = dtrain['interest_level']\n",
    "\n",
    "train = dtrain.drop(['interest_level'], axis=1)\n",
    "X_train = np.array(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, cv_folds=None, early_stopping_rounds=10):\n",
    "    xgb_param = alg.get_xgb_params()\n",
    "    xgb_param['num_class'] = 3\n",
    "    \n",
    "    xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "    cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "             metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "  \n",
    "    cvresult.to_csv('1_nestimators.csv', index_label = 'n_estimators')\n",
    "    \n",
    "    n_estimators = cvresult.shape[0]\n",
    "    \n",
    "#    print n_estimators\n",
    "    \n",
    "    alg.set_params(n_estimators = n_estimators)\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "\n",
    "    print (\"logloss of train :\" )\n",
    "    print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss of train :\n",
      "0.5285797068040592\n"
     ]
    }
   ],
   "source": [
    "xgb1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=1000,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb1, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pxt/anaconda2/lib/python2.7/site-packages/ipykernel_launcher.py:1: FutureWarning: from_csv is deprecated. Please use read_csv(...) instead. Note that some of the default arguments are different, so please refer to the documentation for from_csv when changing your function calls\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcHHW97//Xp7tnX5PMZJ1sLGGTsIVdhAMugGyiIigq6JUrv4PL8agHr/44HK4IiscNuSp6geMGorgAoqKsetiSAAECBJKQfV8ms6/9uX98a2Y6k5nMJEx3zaTfz8ejHt1dXV31meqefnfVt+pb5u6IiIgAJOIuQERERg+FgoiI9FIoiIhIL4WCiIj0UiiIiEgvhYKIiPRSKIhkMLP/ZWY/ibsOkbgoFMYYMys3sxVm9sGMcRVmtsrM3pcxbp6Z3W9m282s3sxeNrPrzWxc9PxlZtZtZk3RsNzMrsxy7aeZ2ZpsLmNPDFSPu3/N3f9Hlpa3wszeno15Z0Ou3q+xtl72dQqFMcbdm4ArgO+aWW00+hvAAnf/DYCZnQQ8Cvw3cLC7VwNnAl3AERmze9Ldy929HHgf8A0zOyo3f4nsCTNLxV2D5Al31zAGB+AO4E7gNGArMCXjuX8ANw/x+suAf/Qb9wzwwYzH5wGLgXpCyByS8dwh0bj6aJrzMp47G3gZaATWAp8HyoBWIA00RcPUQf6uW4A/Rq9/Gth/GOvjYOCvwDZgCXDR3tQDXAv8PHrdLMCBy4HVwHbgk8CxwAvR3/79jOXsDzwcvR9bgF8A1dFzP4uW1Rot64vDWMcrgH+LltUOpKLHa6O/ZQlwxgDr4gRgA5DMGPce4IXo/nHAAqAB2Ah8a5B1ehqwZpDnqoCfApuBlcBXgET0XBL4z2gdvAFcFa3H1CDzWgG8fZDnPgEsjd7Xe3s+M4AB3wY2ATuidfSWwd7vuP9fx9IQewEa9vKNg3HA+ugf7/KM8WVAN3DaEK+/jIxQiL7o6oE50eM5QDPwDqAA+GL0z1kYPV4K/K/o8enRP+BB0WvXA6dk1Hl0dH/QL5mMOu6IvgCOi74EfwHcNcRryghf2pdHrzk6Wi+H7Wk9DBwKPwSKgXcCbcDvgYnAtOhL6dRo+gOi9VUE1AKPA9/JmPdOX367W8cZ0z8PTAdKgIOiv3NqRn0DBiawDHhHxuNfA1dH958EPhzdLwdOGGQeg75fhED4A1AR1fEa8PHouU8SvpTrovX9N/YiFKLP1Zbo/SwCbgYej557F7AQqCYExCFEP4wGe781DG/Q7qMxyt23E35hlgK/zXhqHGG34IaeEWb2jahdodnMvpIx7QnR+CbCVsLPgNej5z4A/NHd/+runcA3CV9MJxF+iZYDN7p7h7s/DNwPXBK9thM41Mwq3X27uz+7h3/eb939GXfvIoTCkUNMfw6wwt1vd/euaHn3EHaJjUQ9/9vd29z9QcKX+J3uvsnd1wJ/B44CcPel0fpqd/fNwLeAU3cz392t4x7fc/fV7t5KCPui6G8pcPcV7r5skHnfSfR+mFkF4dfznRnr4wAzq3H3Jnd/ak9Whpklo9q/5O6N7r6CsGXw4WiSi4Dvuvua6HN6457MP8OHgNvc/Vl3bwe+BJxoZrOiv6GCsIVo7v6Ku6/P+PvezPud1xQKY5SZXUr4hfY34OsZT20n7KaY0jPC3b/ooV3hd4Rf0j2ecvdqD20Kk4HDgK9Fz00l7BbomUea8Ct1WvTc6mhcj5XRcwDvJXwJrTSzx8zsxD388zZk3G8hBNDuzASOjwKu3szqCV8ok0eono0Z91sHeFwOYGYTzewuM1trZg3Az4Ga3cx3d+u4x+qM55cCnyVszWyKljV1kHn/ErjQzIqAC4Fn3b1nWR8nbKW8ambzzeyc3dQ4kBrCFuLKjHGZ7//UzLr73d8T/ddPE2HX3LToh8j3CbsaN5rZrWZWGU36Zt/vvKZQGIPMbCJhf+ongP8JXGRmbwNw92bCfvgL92Se7r6R8Ov63GjUOsKXbc8yjbAbY2303HQzy/z8zIiew93nu/v5hF0svwfu7lnMntS0B1YDj0UB1zOUu/uVOa7nhmiec929EriUsGujR//l7W4dD/gad/+lu781ep2z8w+CzOleJnyhngV8kBASPc+97u6XENbH14HfmFnZ8P9MthB+jc/MGNf7/hN239RlPDd9D+adqf/6KQMm0Pc5+567H0P4MTMH+EI0frD3W4ZBoTA2fR/4vbs/Em0yfxH4cfSrkOjxx8zs6ihAMLM6YPZgMzSzCYTGyMXRqLuBd5vZGWZWAPwrobHzCULoNANfNLMCMzuNECZ3mVmhmX3IzKqiXSINhN0eEH5hTzCzqhFaDz3uB+aY2YejegrM7FgzOyTH9VQQGpHrzWwa0ZdUho3AfhmPd7eOd2FmB5nZ6dH73EbYSukeaNrIL4FPA28jtCn0zOdSM6uNtkzqo9GDzsfMijMHwpbo3cD10eHQM4HPEbaMev6uz5jZNDOrJjSOD6Wg33JSUf2Xm9mR0d/8NeBpd18Rvb/HR+utOVof3UO83zIccTdqaNizAbiA8Auqut/4h4DrMx4fDzxA+KevB14CrgcmRM9fRvhn6TnyZhNhn/PEjHm8h9BguAN4jKjhNnrusGjcjmia90TjC4E/E3ZjNQDzgbdmvO42wi6AegY/+uirGY9PY4jG6Wi6gwhHLG2O5v8woS1ij+ph4IbmVMb0a8hoxCd8EX4lY50sjNbn84Qv+TUZ054PrIqW9flhrOMV7NwwPZfQ9tNIaIy/f6B1mDH9DMIX+B/7jf959H43EX4EXDDI60+L/v7+wwGEtqufR+t7NXANfUcfpQhbslsJRx/9C2HLwgZZzooBlvHV6LlPEhrNe/7eumj8GYQjjproO9KrfKj3W8PQg0UrWEQkK8zsLOCH7j5zyIkldtp9JCIjysxKzOxsM0tFu9H+nXCQg4wB2lKQMcHMTgH+NNBzHo6eklHCzEoJu8IOJrR7/BH4jLs3xFqYDItCQUREemn3kYiI9BpznWzV1NT4rFmz4i5DRGRMWbhw4RZ3rx1qujEXCrNmzWLBggVxlyEiMqaY2cqhp9LuIxERyaBQEBGRXgoFERHppVAQEZFeCgUREemlUBARkV4KBRER6ZU3obBsUyP3zX8NdeshIjK4vAmFLX++kXP/eCzNLS1xlyIiMmrlTSikysYDsGPLhiGmFBHJX3kTCoWVocuPxm3rY65ERGT0yptQKKqaCEDrjk0xVyIiMnrlTSiUj5sMQPuOzTFXIiIyeuVNKFTWhFDoatoScyUiIqNX3oRCWWUNaTdoViiIiAwmb0LBkil2WAWJ1m1xlyIiMmrlTSgANCYqKWxXKIiIDCavQqGloJqizvq4yxARGbXyKhTaC8ZR1qVQEBEZTF6FQmfxeCrTO+IuQ0Rk1MpaKJjZbWa2ycxeGuR5M7PvmdlSM3vBzI7OVi09vGQ81TTS3tmZ7UWJiIxJ2dxSuAM4czfPnwUcGA1XAD/IYi0AWFkNKUuzY/vWbC9KRGRMyloouPvjwO4O9Tkf+KkHTwHVZjYlW/UAFFSE/o8a1CmeiMiA4mxTmAaszni8Jhq3CzO7wswWmNmCzZv3vpuKosrQ/1Fz/ca9noeIyL4szlCwAcYNeAUcd7/V3ee5+7za2tq9XmDZuBAK7eoUT0RkQHGGwhpgesbjOmBdNhdYMSHsnepsVKd4IiIDiTMU7gU+Eh2FdAKww92zerGDivGTAEirUzwRkQGlsjVjM7sTOA2oMbM1wL8DBQDu/kPgAeBsYCnQAlyerVp6JIvKaPVCbO38bC9KRGRMyloouPslQzzvwD9na/mDqU+OJ1VckevFioiMCXl1RjNAY8EESjq0+0hEZCB5FwptRTVUdKmnVBGRgeRdKHSW1DIuvZ2w90pERDLlXShQNolx1kR9Q1PclYiIjDp5FwqpqnBY6rbNWT0lQkRkTMq7UCgeF05ga9yyNuZKRERGn7wLhfIJdQC0blcoiIj0l3ehUD0x9LnXUa+eUkVE+su7UCgdNxkAb1RPqSIi/eVdKFiqiHoqSLaoUzwRkf7yLhQAGpLjKWxTKIiI9JeXodBSOIGyDl2SU0Skv7wMhfbiGqrS2+MuQ0Rk1MnLUHi5sYQa305bR1fcpYiIjCp5GQqHzplDiXWwabMuyykikikvQ6F4fDiBbeuGFfEWIiIyyuRlKFRMmglA86aVMVciIjK65GUojJ88G4CO7WtirkREZHTJy1AoHj+NNIbvUP9HIiKZ8jIUSBaw3cZR2Lw+7kpEREaV/AwFYEdBLaVt6v9IRCRT3oZCa8kkqrvU1YWISKa8DYWu8qlM9K20dnTHXYqIyKiRt6GQqJpGhbWyQSewiYj0yttQKIpOYNu+fkWsdYiIjCZ5GwoVE2cB0LRZJ7CJiPTI21AYPyU6gW3b6pgrEREZPfI2FIrGhWs1+/LHYq5ERGT0yGoomNmZZrbEzJaa2dUDPD/TzB4ysxfM7FEzq8tmPTtJFbI1MYHCouKcLVJEZLTLWiiYWRK4BTgLOBS4xMwO7TfZN4Gfuvtc4DrghmzVM5AdhVOoaNNZzSIiPbK5pXAcsNTdl7t7B3AXcH6/aQ4FHoruPzLA81nVVjaN2u6NdHWnc7lYEZFRK5uhMA3IbMVdE43LtAh4b3T/PUCFmU3oPyMzu8LMFpjZgs2bR+4s5HT1DKawlQ31TSM2TxGRsSyboWADjPN+jz8PnGpmzwGnAmuBXa6R6e63uvs8d59XW1s7YgUW1cwmZWk2rlkxYvMUERnLshkKa4DpGY/rgHWZE7j7One/0N2PAr4cjduRxZp2Ujl5PwAa1r+eq0WKiIxq2QyF+cCBZjbbzAqBi4F7Mycwsxoz66nhS8BtWaxnF+OmzQGgfcuKXC5WRGTUyloouHsXcBXwF+AV4G53X2xm15nZedFkpwFLzOw1YBJwfbbqGUjh+OmkMax+VS4XKyIyaqWyOXN3fwB4oN+4azLu/wb4TTZr2K1UIdsSEyhq0mU5RUQgj89o7tFQNJXKdp2rICICCgWWtI9nYnojbZ26roKISN6Hwn6F25nCVlZvro+7FBGR2OV9KJQc91GS5mxY9VrcpYiIxC7vQ2H8jEMAaFq7JOZKRETil/ehUD4lnKvQvWVpzJWIiMQv70OB0gk0WxkFO96IuxIRkdgpFMzYVjSd6lZdgU1ERKEAtFbOZGr3Oprbd+mLT0QkrygUAJuwP1NtCys3bYu7FBGRWCkUgNJJc0ias2mVeksVkfymUAAmzDwYgD89+o+YKxERiZdCASiedBAAJ1TrrGYRyW8KBYDS8TQmKinasSzuSkREYqVQiGwv249JbSvo6k7HXYqISGwUCpHO8XPY39awcmtz3KWIiMRGoRApnnIo1dbMylUr4i5FRCQ2CoXIhNmHA1C/8qWYKxERiY9CIVI85TAAuja+HHMlIiLxUSj0qJhMi5VRXK/eUkUkfykUepixjDpqW1fQ0aUjkEQkPw0ZCma2v5kVRfdPM7NPm1l19kvLvcoZb+GAxBqWbmqKuxQRkVgMZ0vhHqDbzA4A/i8wG/hlVquKSWndW6i1Bpa+sTzuUkREYjGcUEi7exfwHuA77v4vwJTslhWP8fsdA0D9G8/FXImISDyGEwqdZnYJ8FHg/mhcQfZKik9ySjgs1Ta+GHMlIiLxGE4oXA6cCFzv7m+Y2Wzg59ktKyal46kvmMS4hiWk0x53NSIiOZcaagJ3fxn4NICZjQMq3P3GbBcWl+ZxhzBnw+us2tbCrJqyuMsREcmp4Rx99KiZVZrZeGARcLuZfSv7pcWjYOpc9rP1vLJ6U9yliIjk3HB2H1W5ewNwIXC7ux8DvH04MzezM81siZktNbOrB3h+hpk9YmbPmdkLZnb2npU/8sbtfzQpS3P3Aw/GXYqISM4NJxRSZjYFuIi+huYhmVkSuAU4CzgUuMTMDu032VeAu939KOBi4P8Md/7ZUjDtCABOKF0bcyUiIrk3nFC4DvgLsMzd55vZfsBwLmZ8HLDU3Ze7ewdwF3B+v2kcqIzuVwHrhld2FlXPoi1ZTtX2xXTq2goikmeGDAV3/7W7z3X3K6PHy939vcOY9zRgdcbjNdG4TNcCl5rZGuAB4FMDzcjMrjCzBWa2YPPmzcNY9JuQSNA4/nAO53VeXd+Y3WWJiIwyw2lorjOz35nZJjPbaGb3mFndMOZtA4zrf5znJcAd7l4HnA38zMx2qcndb3X3ee4+r7a2dhiLfnOKZh3HQbaaF1bEv+EiIpJLw9l9dDtwLzCV8Ev/vmjcUNYA0zMe17Hr7qGPA3cDuPuTQDFQM4x5Z1XF/ieQsjRbX3sm7lJERHJqOKFQ6+63u3tXNNwBDOfn+nzgQDObbWaFhIbke/tNswo4A8DMDiGEQpb3Dw3N6uYBkFz/bMyViIjk1nBCYYuZXWpmyWi4FNg61Iui/pKuIjRSv0I4ymixmV1nZudFk/0r8AkzWwTcCVzm7vGfSlw+kcbiKcxse4VNDW1xVyMikjNDntEMfAz4PvBtQpvAE4SuL4bk7g8QGpAzx12Tcf9l4OThFptL3VOO4cjlT/HMim2cM3dq3OWIiOTEcI4+WuXu57l7rbtPdPcLCCey7dMqDjyJOtvCq0tejbsUEZGc2dsrr31uRKsYhZIzTwSg440nY65ERCR39jYUBjrcdN8yeS6diRKmNjxPfUtH3NWIiOTE3oZC/I3B2ZZM0TLpaI5NLGH+iu1xVyMikhODhoKZNZpZwwBDI+GchX1e2QEnc7Ct4obfPRV3KSIiOTHo0UfuXpHLQkaj1KyT4O/OvNSyuEsREcmJvd19lB/qjiVNglmNz7Fme0vc1YiIZJ1CYXeKymmffAwnJV7i8de2xF2NiEjWKRSGUHzQ6cxNvMHCV5fHXYqISNYpFIZg+59OAqfltUd1fQUR2ecNp+vsgY5CWh11p71fLoqM1bRj6EqVcpK9yDNvbIu7GhGRrBrOlsK3gC8Qus2uAz4P/JhwJbXbslfaKJEswGadwinJl3hw8Ya4qxERyarhhMKZ7v4jd2909wZ3vxU4291/BYzLcn2jQvLAtzPLNvDKS88xGjpxFRHJluGEQtrMLjKzRDRclPFcfnxDznkXAIe3PMXidQ0xFyMikj3DCYUPAR8GNkXDhwnXVS4hXC9h3zduJl01h3BG4lmu+OmCuKsREcmaIa+n4O7LgXMHefofI1vO6JU6+CyO2/JdyryZ7rSTTOz7fQKKSP4ZztFHddGRRpvMbKOZ3WNmdbkoblSZcyYpujm46WmeWKYT2URk3zSc3Ue3E66tPJVwBNJ90bj8UjcPL5vEeYXzuWfhmrirERHJiuGEQq273+7uXdFwB1Cb5bpGn0QSO+wCTuE5Hlq0jIa2zrgrEhEZccMJhS1mdqmZJaPhUmBrtgsbld5yIcV0cLot5IEX1sddjYjIiBtOKHwMuAjYAKwH3gdcns2iRq264/DKOi4umc89z2oXkojse4YMBXdf5e7nuXutu0909wuAC3NQ2+iTSGA4x3UvZNmKlazY0hx3RSIiI2pvO8T73IhWMZZ86NckSXNe8gku+bGuyCYi+5a9DYX8PUh/0mEw5Qg+Vv4ELR3dtHR0xV2RiMiI2dtQyI/uLQZz5IeY0b6UqW1LuefZtXFXIyIyYgYNhUG6zG4ws0bCOQv56y3vwzE+mbqP6+9/me50fmekiOw7Bg0Fd69w98oBhgp3H7J7jH1a2QRs7gd4d9HzpLqaeOBFHZ4qIvsGXXltbx13BamuFj5Z/Qw3P/w6aW0tiMg+IKuhYGZnmtkSM1tqZlcP8Py3zez5aHjNzOqzWc+IqjsGCsu5rOMuXt/YwDu/83jcFYmIvGlZCwUzSwK3AGcBhwKXmNmhmdO4+7+4+5HufiRwM/DbbNWTFed+l/J0Ax8ZF9oVdA1nERnrsrmlcByw1N2Xu3sH4fKd5+9m+kuAO7NYz8g79AKonsnnSh/gjS1N/PLpVXFXJCLypmQzFKYBqzMer4nG7cLMZgKzgYezWM/IS6bgpE9RtfV5/qn4df73/S+zrbkj7qpERPZaNkNhoBPcBmuNvRj4jbt3DzgjsyvMbIGZLdi8efOIFTgijroUkgXcUvA9zJyv3v9y3BWJiOy1bIbCGmB6xuM6YN0g017MbnYdufut7j7P3efV1o6yXrsLSuCsb1DauY2bDl/Pb59by7u+/VjcVYmI7JVshsJ84EAzm21mhYQv/nv7T2RmBwHjgCezWEt2HfVhSJVw3tL/n9ICWLG1Rd1fiMiYlLVQcPcu4CrgL8ArwN3uvtjMrjOz8zImvQS4y93H7oH+yQJ4zw9JdLVw/ylraO9K860HX4u7KhGRPWZj7bt43rx5vmDBgrjL2JU7/OQMaNzAKS03sboJfv7x43nrgTVxVyYigpktdPd5Q02nM5pHihm8/VpoWMtDZV/mwInlfPZXz7OpoS3uykREhk2hMJJmvw0OPofC5g386NxatjW3c9o3H6WpXe0LIjI2KBRG2pk3ghn7zb+On3zkGFo6ujn5xod1trOIjAkKhZFWPR3KJsJrf+L0jseYXVPGjtZOrr7nRcZa+42I5B+FQjZ8+lmYfjw88AUe+cQBfPbtB3LPs2s48YaHFAwiMqopFLIhkYQLfgAdTXDL8Xzm1OlcfvIsNjS08x/3vaxutkVk1FIoZMuE/eGin0JHE3bfZ7nm3YcwubKIO55YwZW/WEhrx4A9eoiIxEqhkE2HnANVM+CFu7Cn/g9PfukMrjnnUP6yeCPzvvpXNjXqcFURGV0UCtn2mUVwyLnw4JexH5zEx946mx9/ZB6tnd289cZHWLKhMe4KRUR6KRSyLZGAC34Ikw+H7Sth3fO849BJ/OGf30p1aQFnf/fvvONb6kBPREYHhUIuFJXDB38N6S74ydth48scXlfFH646meKCBK9vauLY6//GjpbOuCsVkTynUMiVyilw5RNQVgM/OgV+dCpTqkp47pp3MqWqmM2N7Rzz1b9y+jcf1WGrIhIbhUIu1RwAl/0RLAkbX4J1z1GYSvDkl87gj59+K8UFSZZvaebwax/kmTe2xV2tiOQhhUKuTdgfrnomBMOPz4A1CwE4bGoVL/z7O/nGe+dSVpTkoh89yZH/8SCL1+2IuWARySfqOjsu9avhv84Jt7WHwP/3371PtXZ0819PruCmvyyhO+2ce8RUrjx1fw6dWhlfvSIypg2362yFQpwa1sHNx0BnC7zrBjjhytAFd2RHaye3Pr6MHzy6jLRDZXGKb77/CM44ZBLJxECXwBYRGZhCYaxob4TvHAGtW+GIS+Dd34LC0p0m2dHSyTk3/52NDe10dKcpSiX4tzMP5v3z6qgoLoipcBEZSxQKY0k6DY/fBI9+DVLF8JF7Ycbxu0zW1Z3mwZc38sXfvEBTexcJg/Flhdz0/iN46wE1FCTVRCQiA1MojEXLH4NfvB+62+GkT8E/fQUKigecdNHqen7x9ErueXYt3WknlTDeP6+Osw+fwgn7TVBAiMhOFApjVXsjPPgVWHgHpErgsvuhbvD3sb2rm3Nv/gdbmzrY3tJB2iGZMM6dO4V3HjaZU+fUUlaUyl39IjIqKRTGuqUPwZ0XQ3cHHHM5nHENlI7f7UtaO7o5//v/YHtLB9tbOulKOwaUF6WoKE5xw3vncszMcZQrJETyjkJhX9DWAI/eCE/dAokUnH0THPVhSA7duNzVnebcm/9BfWsnDW2dNLf3ddWdMJhYUcS/nXUwh0ypZP/acu1uEtnHKRT2JRtegtvPgvaG0BBdNQP++alwMZ9ham7v4rlV9fzbPYvYsKONNNDz1htQUpikozvNtKoSvvG+uRw8pZKqEh3ZJLKvUCjsa9zh9Qfh15dDZzMUlITeVw85L/TEuoe6utMs39LMlT9fSEtHN83tXTS2dZH5aShKJSgtTFJamOSacw/j0CmV1I0rwUznSIiMNQqFfVU6Da/cC498DbYsgcIyuPDHMOesvQqHTO7O5sZ2Fq9v4Jrfv0RzRzctHV20daZ3mq6iOEVpYZKSgiTXnncY08eXUjeuhKLU8LdcRCS3FAr7unQ3vPhruO8z0NUWjlR61/Vw5AfDVsQIaunoYsmGRj73q+dZU99KUSpJS0cX/S81XZA0ilNJWju7mVRZTFEqQXFBgtsuO46JFUUkdBa2SGwUCvmiuwt+cBI0rIWOptAgfcq/wtEfhappWVtsOu1sbmpn9bYWvvibF1hT30plcYr2zjRN7TvvhoLQe4cBFcUFFCYT7GjtYGp1CV9596FMripmYmURNWUKDpFsUSjkG3dY+QT86lJojbrdLhkH598CB74Lkrk9DLW9q5u121u58ucLae9K096VZnNjO0WpBB3daTq7d/3c9cRBaVGSwmSCpvYuJlUWU5hMUJhK8P0PHsWkymLKi1Jq1xDZQ6MiFMzsTOC7QBL4ibvfOMA0FwHXAg4scvcP7m6eCoVh2LYcnv0ZPHkzdHdC+WQ48hKYezFMPDju6oDQ0L25qZ2NDe184deLQlB0pdnU2E5pYZLObqe1s3vQ1xckjVQiQUdXN5UlBaQSYetjUmUxV51+ANWlhYwrLWRcaQHVpYVUlxbosFvJa7GHgpklgdeAdwBrgPnAJe7+csY0BwJ3A6e7+3Yzm+jum3Y3X4XCHujuhNf+AvdeBa3bw7jJc+Hw98HB54RrO4xyLR1dbGpo58qfL4y2MNJs2NFGVUkhXek0Da2dFKaSdKUH3vrIlDQjjVNakCSVNJrbuxlfVkgyYWxt7mBqVTGphJFMJPj6++ZSWZxifFkhlcUF2q0lY95oCIUTgWvd/V3R4y8BuPsNGdN8A3jN3X8y3PkqFPZS0yZ46R54+Kuh7QFg4mFwyDkhICYfvlO33WORu9PS0c32lg7qWzrZ3tLBf9y7mNXbW6kpL6IrnWZrUwdlRSm60mlaOrpJmtGd9l3aQPpLJYxud4pTSRIJeo/IqiopIGlQ39pJbXloE9nc2M7U6hKSBmvrW5k1oYwb3juXkoJwxFZJYTQUJNUFuuTMaAiF9wFnuvv/iB5/GDje3a/KmOb3hK2Jkwm7mK519z/vbr4KhRGwfSWD6/FwAAAS6UlEQVS8+kd49IZwQhxA9Uw45NwQENOP26MT48Y6d6etM01DWycN0RngO1o7uf6Pr7BmeysTygvp6na2NXdQUZwi7dDU1okDhakE6TR0dKcxGDJc+jMLzUGFyQSJBHR0pSkrTJFIGM3tXVSXFpAwY3tLB7UVRSTN2NTYzpSqYsyM9TtaqasO546s2d7CzAllJAxWbG1h/9oyvv2BI0kmEqQSRippJBNGQSJBMhluC5JGSrvV8sJoCIX3A+/qFwrHufunMqa5H+gELgLqgL8Db3H3+n7zugK4AmDGjBnHrFy5Mis156WmzbDkAfjbv/ftYiqrDQFx2Htg5sl5FRBvVkdXmpaOLpqjEwJbOrpp7ejm2ntfYuW2FqZWlZB2Z/2ONmrKi0i7s6WpnaqSAtIeLqxUWpgknQ5tKqlkgnTa6ep//G8WpBJGwoyudJrigiQGtHZ2U1aUwoDmjm4qilKYQWPbzoE1oayIhMGW5g4mVRRhZmxsaGNqdQkGrMsIr9XbW5gxvhQDVm0LQWbAim0tzJ5Q1nuk2k3vPyLUlDCSFgJtp8HCc6noccKsd3rZ1WgIheHsPvoh8JS73xE9fgi42t3nDzZfbSlkUVtDOGv6lfvCCXKehrKJMOedMPs0mH0KVEyOu8q85e69R3J1dKXp6A63nd1pPver51m+pZmZ40tJO6za1kzduFLcnTX1rUytKsHdWbejjcmVxTjOhoZ2JkbBtLmpnXGlhaTdqW/ppLw4hTs0tXdRWpjEPbTvFBeE+21d3RQkEjhOZ7eTMHY5byVuRtgS6+k52CB0M58ModHV7RSkEhhhS68omQALwV6USmIGbZ3dISAtdDjZPyAxaGrrorI4BWY0tHZSWVKAEQK+ujR0FbOjtZNxpYUAUYgWAsbW5nZqyoswYEtTO7UVxRiwqak9ClfY2NDO5MpisHD1xfs+dcrerY9REAopwq6hM4C1hIbmD7r74oxpziQ0Pn/UzGqA54Aj3X3rYPNVKORIRwu8/hdY/Ht49b5wshxAYTm87fMw620wZe6wOueT/NGd9p0Cq+e2O+2k3fnXu59n2eZmZteU4Q5vbG1m1vhSHFi5tZkZ0f1V21qY3i/UILTRAEypKsYdNjS0MamyCHfY2BhCzgln5k8oLwLCl+34skLcwxfyuNJwv761o7d/r/qWTqpKCnCgobWTiuJwCHdjWxflRSmcvoDEobmji5KCJE4IjsJU2AXX0ZXuvd/elaYwmcAdOrvTpJKGO3SlnWTCcHfSzh7tdpw1oZRHv/BPe/XexB4KURFnA98htBfc5u7Xm9l1wAJ3v9fCweb/CZwJdAPXu/tdu5unQiEG6W7Y8AIsfxQe/2ZfQ7Ulwu6lGSfCzBOh7lgoqoi1VJGxzN3pTjvd7qTT0B09TkehWlqYoqRw73bnjopQyAaFwijQuAFWPRUNT8D6RX3PTTkihMSME6DuuKyeVS0iw6dQkNxpb4Q186OQeBJW/CO0RwAki+Dgs2H68eGopsna5SQSh+GGgi7BJW9eUQXsf3oYIJw0t+EFWD0fVj8Nq5+Bxb8Lz1kCpp8QLjE67ZgwVNWN+XMkRPYVCgUZecmCvi/8Ez4Zxu1YC2ue6QuKJ26mt3ktUQAHnBGmn3JkaMDWUU4isVAoSG5UTYOq94RzHwC6OmDjS7B2Iax9Nty+lnHeYvmksKtpytzo9ggYN0tbFCJZplCQeKQKYdrRYejR1gAbXgy7ntYvgpf/AEv/2vd8UVXojmPKEX1hUTMn5z3AiuzL1NAso1tnG2xaDOtfiMLiBVj3bF9DtiVg6lEhICYdBrUHw8RDoKwm3rpFRhk1NMu+oaC4r32iR3cXbH09IygWwXM/7TvBDkI7xYwTQkD0BEXtwVA6Pvd/g8gYolCQsSeZCl/yEw+BIz4QxrlD43rY9DJsehU2vxJun/9l38l20BcWtQdD7UFhqDkIyieqvUIEhYLsK8ygcmoYDnh733h32LEGNr0SgmLza7D5VXjhV309xEK4jOm0eVA7JwRGTRQYOlxW8oxCQfZtZlA9PQxz3tk3vmfLYvOSMGyJbl99AJ79acbrE6Fhu/bg0KhdMwdqDgxHQqWKcv7niGSbQkHyU+aWxf79Ohhr3rJzUGxeAssfg0V37jzduFkw4UCYcEC4it2EA8JQOQ0SukaBjE0KBZH+ymrCMOvknce37YCtS2Hrsuh2KWx5HZY91Hc0FISti4mHZgRFRnCooVtGOYWCyHAVV+16JBREu6I2REHxel9obFwML9/LTh0jl4wLWxjVM8LV7qpnwLjZMGE/qJqhcy4kdvoEirxZZlA5JQyz+10ApbszXP60Z8ti27LweOPLsOTP0N3eN20iFUJi/H7RsH/f/eoZ4YQ/kSxTKIhkU7IAag4IQ3/pNDRvgm1vwLbl0bAs3C57BLx75+mrZ4ZdUOP3C1sX42ZC1fQQGCXjdJSUjAiFgkhcEonQ8V/F5HCRokzuocG7NywyAmPB7bsGRmFFtEsqc5jet5tKoSHDpFAQGY3MoLw2DDOO3/k5d2jdDvWr+oYdq/vur/gHdDTu/JrC8nDORVV0eG7PFkbP4/LJOmJKAIWCyNhjFo5iKh0PU4/c9Xl3aKuPQmL1rsGxdkEIlZ1nuvPuqKrpIUQqp0LFlLA1o62NvKBQENnXmIUv8JJx4cS7gbQ3RSGxGnZE4dHzeNnD4WiqXS4nb327pnrCo3erYzpU1qkxfB+gUBDJR0Xlff1HDaSrAxrWhnBoXA9NG8Ntw7oQHG88Fu7vFBwWtioyd09VTw+H2vaMKyzNxV8nb4JCQUR2lSqE8bPDMJie4Ohtz1jdd3/NfHjpHnbZ2iitCYfuVkwNu6Qqo9vMx6UTtJsqRgoFEdk7QwVHujtsXfSGxcpwv3EDNK4L18Vo3rzr65KFoeG7csqugZH5uKg8u39fnlIoiEh2JJLREU91wIkDT9PV0bdrqnE9NKzvu9+4Ppzkt/ThXY+mgnAYbsWkvobwiskZ96Pb8snaZbWHFAoiEp9UYV8vtrvT3hi2MBrW7drO0bgh7K6qX7VzH1Q9iqv6hUU0VEa35ZPCoEZyQKEgImNBUUUYag4cfJqeQ3F7QiPztmFdCJE3/g5NGyDdtevrSyeELYuKSX23vQEytS9A9vH+qfbtv05E8kfmobiDHVUFoXuRli19u6uaNkDjxr7bnutsNG3cNTwsAWUT+xrLM9s9MscVVY7ZxnKFgojkl0QiXH61fOLg53FAFB5bQ6N4w/pdb7e/ASv/O2yd7MJCg/nkt0S7pyaG27Lavt1VPeNGWZuHQkFEZCCJRF9XI7sLj87WXRvJmzZFw8bQK+6a+aEvq11OCCQ0mPcExIC3GYGSLMjan9tDoSAi8mYUlPR1cb473V1ht1XTxp1DI/N242J45b5dOzzs8e5vwbEfH/m/IUNWQ8HMzgS+CySBn7j7jf2evwy4CVgbjfq+u/8kmzWJiMQimeo7dHYonW2hW/XewIhCY+pRWS8za6FgZkngFuAdwBpgvpnd6+4v95v0V+5+VbbqEBEZcwqK+/qZyrFs9pV7HLDU3Ze7ewdwF3B+FpcnIiJvUjZDYRqwOuPxmmhcf+81sxfM7DdmNuAZLGZ2hZktMLMFmzcPcFq8iIiMiGyGwkAH6fZver8PmOXuc4G/Af810Izc/VZ3n+fu82pra0e4TBER6ZHNUFgDZP7yrwPWZU7g7lvdvefK5T8GjsliPSIiMoRshsJ84EAzm21mhcDFwL2ZE5jZlIyH5wGvZLEeEREZQtaOPnL3LjO7CvgL4ZDU29x9sZldByxw93uBT5vZeUAXsA24LFv1iIjI0Mx9gDPsRrF58+b5ggUL4i5DRGRMMbOF7j5vqOmyuftIRETGmDG3pWBmm4GVe/nyGmDLCJYzUkZjXappeEZjTTA661JNw5eNuma6+5CHb465UHgzzGzBcDafcm001qWahmc01gSjsy7VNHxx1qXdRyIi0kuhICIivfItFG6Nu4BBjMa6VNPwjMaaYHTWpZqGL7a68qpNQUREdi/fthRERGQ3FAoiItIrb0LBzM40syVmttTMro6phulm9oiZvWJmi83sM9H4a81srZk9Hw1n57iuFWb2YrTsBdG48Wb2VzN7Pbodl+OaDspYH8+bWYOZfTbX68rMbjOzTWb2Usa4AdeNBd+LPmMvmNnROazpJjN7NVru78ysOho/y8xaM9bXD7NR027qGvT9MrMvRetqiZm9K4c1/SqjnhVm9nw0PifrajffA7F+rnq5+z4/EPpeWgbsBxQCi4BDY6hjCnB0dL8CeA04FLgW+HyM62cFUNNv3DeAq6P7VwNfj/n92wDMzPW6At4GHA28NNS6Ac4G/kToNv4E4Okc1vROIBXd/3pGTbMyp4thXQ34fkWf+0VAETA7+v9M5qKmfs//J3BNLtfVbr4HYv1c9Qz5sqUwKq4C5+7r3f3Z6H4joVfYgS48NBqcT9/1Lf4LuCDGWs4Alrn73p7Jvtfc/XFCZ42ZBls35wM/9eApoLpfT8BZq8ndH3T3rujhU4Su6nNqkHU1mPOBu9y93d3fAJYS/k9zVpOZGXARcOdIL3eImgb7Hoj1c9UjX0JhuFeByxkzmwUcBTwdjboq2jS8Lde7aggXP3rQzBaa2RXRuEnuvh7ChxiYmOOaMl3Mzv+4ca4rGHzdjJbP2ccIvyx7zDaz58zsMTM7JYZ6Bnq/RsO6OgXY6O6vZ4zL6brq9z0wKj5X+RIKw7kKXM6YWTlwD/BZd28AfgDsDxwJrCds0ubSye5+NHAW8M9m9rYcL39QFq7FcR7w62hU3Otqd2L/nJnZlwld0f8iGrUemOHuRwGfA35pZpU5LGmw9yv2dQVcws4/NnK6rgb4Hhh00gHGZW1d5UsoDHkVuFwxswLCB+EX7v5bAHff6O7d7p4mXIFuxDejd8fd10W3m4DfRcvf2LOJGt1uymVNGc4CnnX3jVGNsa6ryGDrJtbPmZl9FDgH+JBHO6Oj3TNbo/sLCfvu5+Sqpt28X3GvqxRwIfCrjFpztq4G+h5glHyu8iUUhrwKXC5E+zD/L/CKu38rY3zm/sH3AC/1f20Wayozs4qe+4QGy5cI6+ej0WQfBf6Qq5r62enXXJzrKsNg6+Ze4CPR0SInADt6dgdkm5mdCfwbcJ67t2SMrzWzZHR/P+BAYHkuaoqWOdj7dS9wsZkVmdnsqK5nclUX8HbgVXdf0zMiV+tqsO8BRsvnKtst7aNlILTgv0ZI/y/HVMNbCZt9LwDPR8PZwM+AF6Px9wJTcljTfoSjQBYBi3vWDTABeAh4PbodH8P6KgW2AlUZ43K6rgiBtB7oJPxi+/hg64awmX9L9Bl7EZiXw5qWEvY793yufhhN+97ofV0EPAucm+N1Nej7BXw5WldLgLNyVVM0/g7gk/2mzcm62s33QKyfq55B3VyIiEivfNl9JCIiw6BQEBGRXgoFERHppVAQEZFeCgUREemlUBARkV4KBZFhMLMj+3X7fJ6NUBfsFroELx2JeYm8WTpPQWQYzOwywklDV2Vh3iuieW/Zg9ck3b17pGsR0ZaC7FOiC6W8YmY/ji5g8qCZlQwy7f5m9ueod9i/m9nB0fj3m9lLZrbIzB6Puka5DvhAdPGVD5jZZWb2/Wj6O8zsB9GFU5ab2alRj6CvmNkdGcv7gZktiOr6j2jcp4GpwCNm9kg07hILFz16ycy+nvH6JjO7zsyeBk40sxvN7OWoB9JvZmeNSt7J5unSGjTkeiBcKKULODJ6fDdw6SDTPgQcGN0/Hng4uv8iMC26Xx3dXgZ8P+O1vY8JXSbcReiO4HygATic8KNrYUYtPd0WJIFHgbnR4xVEFzkiBMQqoBZIAQ8DF0TPOXBRz7wI3UNYZp0aNLzZQVsKsi96w92fj+4vJATFTqJui08Cfm3hcow/IlwRC+C/gTvM7BOEL/DhuM/dnRAoG939RQ89gy7OWP5FZvYs8BxwGOFqW/0dCzzq7ps9XDTnF4SrhwF0E3rWhBA8bcBPzOxCoGWXOYnshVTcBYhkQXvG/W5goN1HCaDe3Y/s/4S7f9LMjgfeDTxvZrtMs5tlpvstPw2kop5APw8c6+7bo91KxQPMZ6C+83u0edSO4O5dZnYc4ap0FwNXAacPo06R3dKWguQlDxc1ecPM3g+9F0c/Irq/v7s/7e7XAFsIfdk3Eq6nu7cqgWZgh5lNIlwnokfmvJ8GTjWzmqgb50uAx/rPLNrSqXL3B4DPEi5iI/KmaUtB8tmHgB+Y2VeAAkK7wCLgJjM7kPCr/aFo3Crg6mhX0w17uiB3X2RmzxF2Jy0n7KLqcSvwJzNb7+7/ZGZfAh6Jlv+Auw90LYsK4A9mVhxN9y97WpPIQHRIqoiI9NLuIxER6aXdR7LPM7NbgJP7jf6uu98eRz0io5l2H4mISC/tPhIRkV4KBRER6aVQEBGRXgoFERHp9f8A2eD/fPvUqbgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7d1bf5bc10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "cvresult = pd.DataFrame.from_csv('1_nestimators.csv')\n",
    "        \n",
    "# plot\n",
    "test_means = cvresult['test-mlogloss-mean']\n",
    "test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "train_means = cvresult['train-mlogloss-mean']\n",
    "train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "x_axis = range(0, cvresult.shape[0])\n",
    "        \n",
    "pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "pyplot.xlabel( 'n_estimators' )\n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'n_estimators.png' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "橙色为训练集，蓝色为测试集"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 粗调，步长为2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [3, 5, 7, 9], 'min_child_weight': [1, 3, 5]}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = range(3,10,2)\n",
    "min_child_weight = range(1,6,2)\n",
    "param_test2_1 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.60306, std: 0.00315, params: {'max_depth': 3, 'min_child_weight': 1},\n",
       "  mean: -0.60257, std: 0.00309, params: {'max_depth': 3, 'min_child_weight': 3},\n",
       "  mean: -0.60262, std: 0.00292, params: {'max_depth': 3, 'min_child_weight': 5},\n",
       "  mean: -0.59063, std: 0.00362, params: {'max_depth': 5, 'min_child_weight': 1},\n",
       "  mean: -0.59046, std: 0.00375, params: {'max_depth': 5, 'min_child_weight': 3},\n",
       "  mean: -0.59031, std: 0.00326, params: {'max_depth': 5, 'min_child_weight': 5},\n",
       "  mean: -0.58983, std: 0.00403, params: {'max_depth': 7, 'min_child_weight': 1},\n",
       "  mean: -0.58861, std: 0.00368, params: {'max_depth': 7, 'min_child_weight': 3},\n",
       "  mean: -0.58825, std: 0.00355, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.59992, std: 0.00529, params: {'max_depth': 9, 'min_child_weight': 1},\n",
       "  mean: -0.59577, std: 0.00409, params: {'max_depth': 9, 'min_child_weight': 3},\n",
       "  mean: -0.59398, std: 0.00390, params: {'max_depth': 9, 'min_child_weight': 5}],\n",
       " {'max_depth': 7, 'min_child_weight': 5},\n",
       " -0.5882463666520695)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=205,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_1 = GridSearchCV(xgb2_1, param_grid = param_test2_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_1.grid_scores_, gsearch2_1.best_params_,     gsearch2_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([149.30105257, 154.10949879, 152.74204755, 242.80049176,\n",
       "        242.25736318, 240.75765319, 332.94951296, 332.10716   ,\n",
       "        329.52243552, 426.42650895, 408.01931944, 359.54159393]),\n",
       " 'mean_score_time': array([0.67085681, 0.58957605, 0.66445665, 0.98059845, 0.94760556,\n",
       "        0.96379285, 1.59144497, 1.51542563, 1.48397341, 2.76117435,\n",
       "        1.97007718, 1.42061138]),\n",
       " 'mean_test_score': array([-0.60305717, -0.60256924, -0.60261954, -0.59062754, -0.59046148,\n",
       "        -0.59031279, -0.58982764, -0.58861361, -0.58824637, -0.59992465,\n",
       "        -0.59577036, -0.59397767]),\n",
       " 'mean_train_score': array([-0.58227088, -0.58281357, -0.58311422, -0.51917806, -0.52359257,\n",
       "        -0.52713352, -0.42124126, -0.44004636, -0.45252716, -0.30280298,\n",
       "        -0.34638593, -0.37226852]),\n",
       " 'param_max_depth': masked_array(data=[3, 3, 3, 5, 5, 5, 7, 7, 7, 9, 9, 9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_min_child_weight': masked_array(data=[1, 3, 5, 1, 3, 5, 1, 3, 5, 1, 3, 5],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'max_depth': 3, 'min_child_weight': 1},\n",
       "  {'max_depth': 3, 'min_child_weight': 3},\n",
       "  {'max_depth': 3, 'min_child_weight': 5},\n",
       "  {'max_depth': 5, 'min_child_weight': 1},\n",
       "  {'max_depth': 5, 'min_child_weight': 3},\n",
       "  {'max_depth': 5, 'min_child_weight': 5},\n",
       "  {'max_depth': 7, 'min_child_weight': 1},\n",
       "  {'max_depth': 7, 'min_child_weight': 3},\n",
       "  {'max_depth': 7, 'min_child_weight': 5},\n",
       "  {'max_depth': 9, 'min_child_weight': 1},\n",
       "  {'max_depth': 9, 'min_child_weight': 3},\n",
       "  {'max_depth': 9, 'min_child_weight': 5}],\n",
       " 'rank_test_score': array([12, 10, 11,  6,  5,  4,  3,  2,  1,  9,  8,  7], dtype=int32),\n",
       " 'split0_test_score': array([-0.59796634, -0.59735182, -0.59802364, -0.58435515, -0.58414693,\n",
       "        -0.58521771, -0.58304743, -0.58186511, -0.58259428, -0.59028787,\n",
       "        -0.59025027, -0.5891077 ]),\n",
       " 'split0_train_score': array([-0.58392128, -0.5847999 , -0.58469243, -0.52120528, -0.5252434 ,\n",
       "        -0.52826068, -0.42274696, -0.44192794, -0.45357563, -0.3024851 ,\n",
       "        -0.34633049, -0.37199063]),\n",
       " 'split1_test_score': array([-0.60189921, -0.60116794, -0.6008299 , -0.58962101, -0.58899253,\n",
       "        -0.58876812, -0.58932691, -0.5877605 , -0.58644331, -0.60008753,\n",
       "        -0.59169966, -0.59165405]),\n",
       " 'split1_train_score': array([-0.58267915, -0.58317339, -0.58369763, -0.52002748, -0.52442797,\n",
       "        -0.5284857 , -0.4216518 , -0.44021229, -0.45235475, -0.30453995,\n",
       "        -0.34846117, -0.37322906]),\n",
       " 'split2_test_score': array([-0.60302663, -0.60357804, -0.60313591, -0.59161157, -0.59073243,\n",
       "        -0.59037885, -0.58957509, -0.58995851, -0.58857928, -0.60022073,\n",
       "        -0.59697469, -0.59400231]),\n",
       " 'split2_train_score': array([-0.58231029, -0.58269783, -0.58292475, -0.51821377, -0.52333749,\n",
       "        -0.5263707 , -0.41995657, -0.43908089, -0.45267785, -0.30176095,\n",
       "        -0.34558714, -0.37347075]),\n",
       " 'split3_test_score': array([-0.60735523, -0.60629009, -0.60622859, -0.59228288, -0.5943997 ,\n",
       "        -0.59237284, -0.59179334, -0.59171105, -0.59069343, -0.60293907,\n",
       "        -0.60016232, -0.59430717]),\n",
       " 'split3_train_score': array([-0.58111404, -0.5815727 , -0.58196322, -0.51929642, -0.523149  ,\n",
       "        -0.52643891, -0.41861384, -0.43875507, -0.45061618, -0.30197707,\n",
       "        -0.34473954, -0.37005889]),\n",
       " 'split4_test_score': array([-0.60503905, -0.6044589 , -0.60488037, -0.59526851, -0.59403692,\n",
       "        -0.59482779, -0.59539712, -0.59177386, -0.59292295, -0.60608994,\n",
       "        -0.59976607, -0.6008192 ]),\n",
       " 'split4_train_score': array([-0.58132963, -0.58182405, -0.58229307, -0.51714736, -0.52180498,\n",
       "        -0.52611162, -0.42323715, -0.44025562, -0.45341141, -0.30325183,\n",
       "        -0.34681131, -0.37259327]),\n",
       " 'std_fit_time': array([ 2.54537055,  1.16053306,  0.82411792,  1.80603117,  2.21433726,\n",
       "         1.77126769,  2.5997981 ,  1.2490082 ,  2.18639291,  4.1619585 ,\n",
       "        12.13620054, 16.72869663]),\n",
       " 'std_score_time': array([0.05576848, 0.03983366, 0.10433879, 0.12881754, 0.08334923,\n",
       "        0.05032036, 0.14906317, 0.18240303, 0.11498182, 0.23972593,\n",
       "        0.31351843, 0.0861838 ]),\n",
       " 'std_test_score': array([0.00315155, 0.00308611, 0.00292333, 0.00362217, 0.00375229,\n",
       "        0.00325504, 0.004029  , 0.00367862, 0.00355426, 0.00529294,\n",
       "        0.00409262, 0.00389928]),\n",
       " 'std_train_score': array([0.00101156, 0.00114955, 0.00098652, 0.00140703, 0.00117307,\n",
       "        0.00102054, 0.00173049, 0.00111456, 0.00105693, 0.00100872,\n",
       "        0.00125214, 0.00121927])}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch2_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.588246 using {'max_depth': 7, 'min_child_weight': 5}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAELCAYAAADz6wBxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xt8VNW58PHfM5PJjYRwCfcAIYgKQkCNiFov2CJYMBzUtmLpUVu11XKs7bFVe/NI1WrbV1FLbdXaWlsvVavGFhFUlGqrEGhABBUIqOEaLkkIgUwy87x/7J0wmUySSSaTCfB8P84ns/dea8/aO2Ye1mWvJaqKMcYY01GeRBfAGGPMkc0CiTHGmJhYIDHGGBMTCyTGGGNiYoHEGGNMTCyQGGOMiYkFEmOMMTGxQGKMMSYmFkiMMcbEJCnRBegK2dnZmpubm+hiGGPMEWXlypW7VbVfW+mOiUCSm5tLcXFxoothjDFHFBH5JJp01rRljDEmJhZIjDHGxMQCiTHGmJhYIDHGGBMTCyTGGGNiYoHEGGNMTCyQGGOMickx8RxJR72+fic7q2pJT/aSluwl3X2l+ZIOv0/2kubzkuS1mGyMOTZZIGnF4//+hGUfl0eVNtnraQw2jUHHl9R8X3ISab6QoJScdPi4zz3eGLCc/SlJHkQkzldrjDEdY4GkFQsuP5nq2npq/AEO+gMcrAu47519Dftr/AFq6uob3zs/nTT7avxsrWi6r7Y+2K5yeD3SGFQaAkxjUIpQU2qyLznJDVAN+5OancvjsSBljOk4CyStyEz1kZnq6/TzBoLqBqWQ4FMXEpTC9jcEoMPHAxysq+eAv57d1bUhQa2emroAqu0rT6rPE6GmFBKofE1rT6HBKrSZLzVCgPNZk58xRz0LJAng9QgZKUlkpHT+7VdVauuDTQLS4ZpUSKCqCzQNQE1qV86+nVWHmuWpC7QvSvm84gaoprWl5jWlsEAVkic1QoBLtyY/Y7oNCyRHGREh1ed8+fbpkdzp568LBJs13x2qa1pTalZ7amgKrDucr7LGz/awWtehuvY1+XkEt5mu9ZpSpP6ptJBg1bQp8HA6rzX5GRMVCySmXXxeD1lpHrLSOr/JL9jY5OcGHDcoHQqrKYX3TzUEr9AAt6faH1YTqyfYzia/lCRPs6a6SP1Tae7AivRkL6lNmgKb18QaBmAkJ8W3yU9VUQVteA8E3X3OcVCapgkqEGG/NqQPfR+eRlvY76YH5/xN0oS8J6R8kfKq4uZvWgYa07v5Q/IS9lnBCOcMPVfD+ZvdHzddULXpOZula+lehf4+IucF5///SPeXsN9f5Otvfn8b/j+Ye/4o+mWmdN7/XBFYIDHdhscj9EhJokccm/wORgpIIf1T4fvD+6cO+gOUV9dS469pss8faF9tKskj7og8L4e/KJt/ETX7omjyBdN0OzRQmKOLCHhEEPe9ILj/hR1zfh4+JlxxZq4FEmM6Q2iTX+84nL8+EAxpumujfyqkma+2PtD4x9/wBSEhXwLgfkmEfGk0/bIQPNI8L26aSHkJ+azwvA33ysnrvg87Z6S8oV9cTb7cws4Znpew6w3NG+mcjdcY4V61+L61vG2dJ1LeFn5fRPhCD70/nhbyEnq/wvMeIX2AFkiM6QRJXg89vR56xmGUnzHdnY3NNMYYExMLJMYYY2IS10AiItNE5CMR2Sgit7SQ5ssisk5EPhCRJ0P2XyEiG9zXFe6+dBH5h4h86Ka/O57lN8YY07a49ZGIiBdYAEwByoAVIlKkqutC0owCbgXOUtV9ItLf3d8HuA0owBmQslJEioBa4FequlREkoHXReRCVX0lXtdhjDGmdfGskUwENqpqqar6gaeBmWFprgEWqOo+AFXd5e6fCixR1b3usSXANFWtUdWlblo/sArIieM1GGOMaUM8A8kQ4LOQ7TJ3X6jjgeNF5B0ReVdEpkWbV0R6ARcBr0f6cBG5VkSKRaS4vDy6GXyNMca0XzwDSaQB0OGPSyUBo4DzgNnAo26AaDWviCQBTwEPqGpppA9X1YdVtUBVC/r169eB4htjjIlGPANJGTA0ZDsH2BYhzUuqWqeqm4GPcAJLW3kfBjao6vxOL7Uxxph2iWcgWQGMEpERbsf4ZUBRWJoXgckAIpKN09RVCrwKXCAivUWkN3CBuw8RuQPIAm6MY9mNMcZEKW6BRFXrgbk4AWA98FdV/UBE5olIoZvsVWCPiKwDlgLfV9U9qroX+BlOMFoBzFPVvSKSA/wIGAOsEpESEbk6XtdgjDGmbaLHwCxvBQUFWlxcnOhiGGPMEUVEVqpqQVvp7Ml2Y4wxMbFAYowxJiYWSIwxxsTEAokxxpiYWCAxxhgTEwskxhhjYmKBxBhjTEwskBhjjImJBRJjjDExsUBijDEmJhZIjDHGxMQCiTHGmJhYIDHGGBMTCyTGGGNiYoHEGGNMTCyQGGOMiYkFEmOMMTGJayARkWki8pGIbBSRW1pI82URWSciH4jIkyH7rxCRDe7ripD9p4rI++45HxARiec1GGOMaV1SvE4sIl5gATAFKANWiEiRqq4LSTMKuBU4S1X3iUh/d38f4DagAFBgpZt3H/AQcC3wLrAQmAa8Eq/rMMYY07p41kgmAhtVtVRV/cDTwMywNNcAC9wAgarucvdPBZao6l732BJgmogMAnqq6r/VWWz+T8B/xfEajDHGtCGegWQI8FnIdpm7L9TxwPEi8o6IvCsi09rIO8R939o5jTHGdKG4NW0BkfouNMLnjwLOA3KAf4rI2FbyRnNO58NFrsVpAmPYsGHRlTjM3if+TGB/FVmFhSTn5HToHMYYc7SLZ42kDBgasp0DbIuQ5iVVrVPVzcBHOIGlpbxl7vvWzgmAqj6sqgWqWtCvX78OXcDB99ew+4EH2fSFKWz56hz2/fWvBKqqOnQuY4w5WsUzkKwARonICBFJBi4DisLSvAhMBhCRbJymrlLgVeACEektIr2BC4BXVXU7sF9EJrmjtf4beCleFzDkF7/guNdfo9+NNxLYt48dP72NDZ87m7Lv3Mj+N5aidXXx+mhjjDlixK1pS1XrRWQuTlDwAo+p6gciMg8oVtUiDgeMdUAA+L6q7gEQkZ/hBCOAeaq6131/HfBHIA1ntFZcR2z5hgwh+1vfpO83r+XQ2g+oLCqi6u9/Z/+rr+Lt3ZueX/wiWTMLSR03DhuJbIw5Fokz+OnoVlBQoMXFxZ12Pq2ro/rtt6ksKqL69TdQv5/kESPImllI1kUX4Rti/f/GmCOfiKxU1YI201kgiU2gqoqqV1+l6qUiatzPSC8oIOu/ZpI5dSrezMy4fK4xxsSbBZIQ8QwkofxlW6l6uYjKl4rwb9mCpKSQcf5ksgoLyfjc5xCfL+5lMMaYzmKBJERXBZIGqsqh99+n8qUiqv7xDwIVFXj79KHn9OlkFRaSOvYk608xxnR7FkhCdHUgCaV+v9Of8lIR1W+8gdbVkZyXR9bMmWRdNAPf4MEJKZcxxrTFAkmIRAaSUIHKSqoWvUplUREHV64EEdJPO42smTPJnHoB3oyMRBfRGGMaWSAJ0V0CSSj/Z59RWVREZVERdZ98iqSkkPn5z5P1XzPpceaZSFI8Jx0wxpi2WSAJ0R0DSQNV5dDq1c7zKf9YSKCyEm92NlnTv0jPwkJSx4yx/hRjTEJYIAnRnQNJKPX7qV62zOlPefNNtK6OlFHH0bPQfT5l4MBEF9EYcwyxQBLiSAkkoQIVFVQtWkTlS0Uc/M9/nP6U0093+lOmTMGb0SPRRTTGHOUskIQ4EgNJKP+nn1JZ9DKVL71E3WefIampZH7hC2TNLKTHGWdYf4oxJi4skIQ40gNJA1Xl4H9KqCx6iapXFhGsrMTbL5us6TPImllIyoknWn+KMabTWCAJcbQEklBBv5/qN9905vt6axnU1ZEyahRZ/zWTnjNm4BswINFFNMYc4SyQhDgaA0mo+n37qHrlFapeKuLg6tUgQo8zJjn9KV/4Ap4e1p9ijGk/CyQhjvZAEsq/ZYv7fMrL1JWVIWlpZE75AlkzZ9Jj0iTE6010EY0xRwgLJCGOpUDSQFU5uGqVM9/XK68Q3L+fpP796TnD6U9JPeGERBfRGNPNWSAJcSwGklDB2lqql7r9KcuWQX09KSeeSFZhIT1nTMfXv3+ii2iM6YYskIQ41gNJqPq9e6la+AqVRUUcWrMGPB56nHGGs37K5z+PJz090UU0xnQTFkhCWCCJrLZ0M5UvF1H1UhF127bhSU8nc8oUsmYWkn766dafYswxLtpA4olzIaaJyEcislFEbolw/EoRKReREvd1dcixe0Rkrfv6Ssj+z4vIKjf92yJyXDyv4WiWkjeC/t/5DiNfW8LwJ/5Ez+lfZP/rr/Pp17/BxvM/z65f/YpDH3+c6GIaY7q5uNVIRMQLfAxMAcqAFcBsVV0XkuZKoEBV54blnQ7cCFwIpABvAeerapWIfAzMVNX1InI9MFFVr2ytLFYjiV7w0CGqly515vt6+22nP2XMaLIKC8maPp2kfv0SXURjTBeJtkYSz7k1JgIbVbXULdDTwExgXau5HGOAt1S1HqgXkdXANOCvgAI93XRZwLaOFK6uro6ysjIOHTrUkexHt9xc+M4N6NxvowcPcuDgQarr6ti6Zg2SkoInPR1JTT2inqJPTU0lJycHny13bEyni2cgGQJ8FrJdBpweId0lInIOTu3lu6r6GbAauE1E7gXSgckcDkBXAwtF5CBQBUzqSOHKysrIzMwkNzf3iPpCTJTgoUMEKioJVFagdXUI4MnMxNurF54ePbr1PVRV9uzZQ1lZGSNGjEh0cYw56sSzjyTSN0t4O9rLQK6q5gOvAY8DqOpiYCHwL+Ap4N9AvZvnu8AXVTUH+ANwb8QPF7lWRIpFpLi8vLzZ8UOHDtG3b99u/QXYnXhSU/ENHEDK8ceTnDsCT1YWwaoq/Fu2UPvxx9Tt2EGwm9buRIS+ffta7dOYOIlnICkDhoZs5xDWDKWqe1S11t18BDg15NidqjpBVafgBKUNItIPGK+q77nJngHOjPThqvqwqhaoakG/Ftr1LYi0n4jgzehB8pAhpJx4Ir6cHCQ1lfrde6jduJHajZuo370bra9v+2RdyH7XxsRPPJu2VgCjRGQEsBW4DLg8NIGIDFLV7e5mIbDe3e8FeqnqHhHJB/KBxW66LBE5XlUbOvLXx/EaTCvE4yGpVy+SevVC6+oIVFYSqKigbscO6nbsxJOZQVKvXngyMxFPXAcIGmMSKG5/3W5H+VzgVZwv+7+q6gciMk9ECt1kN4jIB25n+g3Ale5+H/BPEVkHPAzMUdV695zXAM+7eb4GfD9e1xBPFRUV/OY3v+lQ3vnz51NTU9PJJYqN+HwkZWcz9eqreb+ykqTsvuihQ/g/+4zaDz/Ev3UrgeoDtDZK8MUXX2TdusNjMc477zzaGm136NAhJk6cyPjx4znppJO47bbbOu2ajDHRies/E1V1oaoer6ojVfVOd99PVbXIfX+rqp6kquNVdbKqfujuP6SqY9zXJFUtCTnnC6o6zs1zXsOosCPN0RZIQnlSUvANHOj2p+Ti6dmTYGUl/i2bnf6UnTsJ1tY2yxceSKKRkpLCG2+8werVqykpKWHRokW8++67nXUpxpgo2NJ6wO0vf8C6bVWdes4xg3ty20UntXj8lltuYdOmTUyYMIEpU6bQv39//vrXv1JbW8usWbO4/fbbOXDgAF/+8pcpKysjEAjwk5/8hJ07d7Jt2zYmT55MdnY2S5cujXj+jIwMvv3tb/Paa6/Ru3dv7rrrLn7wgx/w6aefMn/+fAoLC9myZQtf+9rXOHDgAAC//vWvOfPMM3nhhRdYsGABS5YsYceOHZx77rksW7aMgRHWjD948CBXXXUV69atY/To0Rw8eLDx2JIlS7jtttuora1lZF4ej8yfT1p9PcedcgqXTp3KspUrwevlySefpHzvXoqKinjrrbe44447eP755wF49tlnuf7666moqOD3v/89Z599dpPPFxEyMjIAZ0h3XV2d9YcY08Ws4TpB7r77bkaOHElJSQlTpkxhw4YNLF++nJKSElauXMmyZctYtGgRgwcPZvXq1axdu5Zp06Zxww03MHjwYJYuXdpiEAE4cOAA5513HitXriQzM5Mf//jHLFmyhBdeeIGf/vSnAPTv358lS5awatUqnnnmGW644QYAZs2axcCBA1mwYAHXXHMNt99+e8QgAvDQQw+Rnp7OmjVr+NGPfsTKlSsB2L17N3fccQevvfYaq1atouC003jgscdIyc1FfD56DRzI288/zzcvvZQbvvlNCoYM4aJp0/jFPfdQUlLCyJEjAaivr2f58uXMnz+f22+/HYBt27bxxS9+sbEMgUCACRMm0L9/f6ZMmcLpp0caZW6MiRerkUCrNYeusHjxYhYvXszJJ58MQHV1NRs2bODss8/mpptu4uabb2bGjBnN/jXemuTkZKZNmwbAuHHjSElJwefzMW7cOLZs2QI4/4KfO3cuJSUleL1ePg6ZDuXBBx9k7NixTJo0idmzZ7f4OcuWLWsMQPn5+eTn5wPw7rvvsm7dOs466ywA/H4/Z5xxRmO+r159NSl5eXztf/6Hm3/1K4IHDxI8cIC6rdvwb92Kt1cvAC6++GIATj311MZyDx48mIULFzaey+v1UlJSQkVFBbNmzWLt2rWMHTs26ntljImNBZJuQFW59dZb+eY3v9ns2MqVK1m4cCG33norF1xwQWNtoi0+n6+xicfj8ZCSktL4vt4dmnvfffcxYMAAVq9eTTAYJDU1tTH/1q1b8Xg87Ny5k2AwiKeVUVeRmpJUlSlTpvDUU0+1mseTmop4PKSccAKejAw86WnO6K99+wjW1OCtriZYW4vX620sd0t69erFeeedx6JFiyyQGNOFrGkrQTIzM9m/fz8AU6dO5bHHHqO6uhpwvsR37drFtm3bSE9PZ86cOdx0002sWrWqWd5YVFZWMmjQIDweD0888QSBQABwmpOuuuoqnnzySUaPHs2990Z85hOAc845h7/85S8ArF27ljVr1gAwadIk3nnnHTZu3AhATU1NkxrPM8880/jzjDPOQETo2acPB5OTST3hBHw5OeDxUF9RQe2GDdRu3gzBYLPnU8rLy6moqACc/prXXnuNE088MeZ7Y4yJntVIEqRv376cddZZjB07lgsvvJDLL7+8seknIyODP//5z2zcuJHvf//7eDwefD4fDz30EADXXnstF154IYMGDWq1n6Qt119/PZdccgnPPvsskydPpoe7tvtdd93F2Wefzdlnn82ECRM47bTTmD59OqNHj252juuuu46rrrqK/Px8JkyYwMSJEwHo168ff/zjH5k9eza17gitO+64g+OPPx6A2tpaTj/9dILBYGOt5bLLLuOaa67hgQce4LnnnsOTmkry8OH4BgyEzaVoIMChjz5i54EDXPfjH7Nw0SK2b9/OFVdcQSAQIBgM8uUvf5kZM2Z0+J4YY9qvzdl/RWQkUKaqtSJyHs7DgX9S1YouKF+niDT77/r16yN+MZr4y83Npbi4mOzs7Hblc+b7qiBQUYHW1yNeL56ePZ35vtLT2xytZb9zY9qnM9cjeR4IuOt+/B4YATwZY/mMaTdnvq+BpJxwAsnDc/FkZhKorMS/eTO1Gza0+HyKMSa+omnaCqpqvYjMAuar6oMi8p94F8xE5/TTT29sOmrwxBNPMG7cuE79nFdffZWbb765yb4RI0bwwgsvtPtcDaOvOkpE8GZm4M3MQAMBAlVVBCorqS8vp768HE9aOt5eWXizspAka701Jt6i+SurE5HZwBXARe4+W9Shm3jvvffaTtQJpk6dytSpU7vks9pDvF6SevcmqXdvgnV1BN2mr7rt26nbsQNvw1T37kOLxpjOF00guQr4FnCnqm52J2H8c3yLZUz7eXw+PP364c3ORhv6UyorCVRVIV4vgYoKav7zH9ImTLCn343pRG0GEndp3BsARKQ3kKmqd8e7YMZ0lIggaWl40tJIGjiQYHU1gYoKgtu388lVX8c3fBhZFxWSVXgRycOGJbq4xhzx2uxsF5E3RaSniPTBWbnwD+7KhcZ0e05/SibJQ4eSNGAAg+66C9+gwexesIBNF0xly+VfZd/TzxCorEx0UY05YkUzaitLVauAi4E/qOqpwBfiWyxjOp94PPS6eBbD//gHjnvjdfp973sEqirZ8X//x4bPnU3Z/9zA/tdfR/3+RBfVmCNKNIEkSUQGAV8G/h7n8hwzjtZp5KNZQ6QlHVmPBJznUsaNG8eECRMoKGhzyDsAvkGDyL72GvJefpnc55+j9+WzqVm1irJvz2XDOeeyY948DpaUtLp+ijHGEU0gmYezONUmVV0hInnAhvgW6+h3tAaSWHRkPZIGS5cupaSkpN1BTERIO+kkBtx6K6PeepOhv/stPc48k4rn/8aWy2ZTOu1Cyn/zG/xlZR0qlzHHgmg6258Fng3ZLgUuiWehutwrt8CO9zv3nAPHwYUtj0k4FtYjWbx48eH1SEaO5A9/+AMZGRnk5ubyla98pbHsTz75JLt27erQeiSdSZKSyDj3XDLOPZfA/v3sX7yYypeK2P3Ag+x+4EHSCk4lq7CQntOm4e3ZM27lMOZIE01ne46IvCAiu0Rkp4g8LyI5XVG4o9kxtx5JQUGTyR979uzJ8uXLmTt3LjfeeCNnnnkmhYWF/PKXv2z3eiQiwgUXXMCpp57Kww8/HMNv5TBvZia9LrmE4X96nONef41+N95IYO8+dvz0Nqc/5Ts3sv+NpWhdXad8njFHNFVt9QUswXmWJMl9XQksaSufm3ca8BGwEbglwvErgXKgxH1dHXLsHmCt+/pKyH4B7gQ+xlkL/oa2ynHqqadquHXr1jXb15U2b96sJ510kqqq/u///q8OHz5cx48fr+PHj9eRI0fqo48+qh999JHm5ubqD37wA122bFlj3uHDh2t5eXmr509OTtZgMKiqqj/5yU/0jjvuUFXVQCCgWVlZqqpaUVGhc+bM0bFjx+r48eM1LS2tMf/evXt18ODBevHFF7f6OTNnztTXX3+9cfvkk0/WFStW6Msvv6x9+/ZtvKbRo0fr17/+9cbyb9q0SVVV/X6/9unTR1VVr7jiCn322Wcbz3Xuuefq22+/raqqO3bs0JEjR0Ysw9atW1VVdefOnZqfn69vvfVWxHSx/s6DwaDWrHlft//sDv1o0hm67oQT9aNJZ+j2eT/TmjVrGu+3MUcLoFij+K6P5oHEfqr6h5DtP4rIjW1lEhEvsACYApQBK0SkSJ3nUkI9o6pzw/JOB04BJgApwFsi8oo6o8euBIYCJ6pqUET6R3EN3Zoew+uRtJS/QUO5W1uPZPDgwYBTw5o1axbLly/nnHPOafGcHSUipI0bS9q4sQy4+QdUv/02lUVFVDz7LPv+8heSR4wga2YhWRddhG/IkE7/fGO6q2g623eLyBwR8bqvOcCeKPJNBDaqaqmq+oGngZlRlmsM8Jaq1qvqAZznV6a5x64D5qlqEEBVd0V5zm7F1iNpuh5JR6/rwIEDjXkOHDjA4sWLu2RRK/H5yJw8mZz77mPU2/9k4M/mkdS3L+Xz72fj57/AJ1/7byqee45AJ/yejOnuogkkX8cZ+rsD2A5citPU1ZYhwGch22XuvnCXiMgaEXlORIa6+1YDF4pIuohkA5NxaiEAI4GviEixiLwiIqOiKEu3E7oeyZIlSxrXIxk3bhyXXnop+/fv5/3332fixIlMmDCBO++8kx//+MfA4fVIJk+eHFMZrr/+eh5//HEmTZrExx9/HHE9knvvvZdHH32U9evXRzzHddddR3V1Nfn5+fziF7+IuB5Jfn4+kyZN4sMPP2zM17Aeyf333899990HOOuR/PKXv+Tkk09m06ZNLZY7tI9k586dfO5zn2P8+PFMnDiR6dOnNy4x3FW8PXvS+0tfYvifn2Dka6/R7zs3UF9ezvYf/8TpT/nud9n/5pvWn2KOWm2uRxIxk8iNqjq/jTRfAqaq6tXu9teAiar6PyFp+gLV6qx18i3gy6p6vnvsR8CXcPpQdgHLVfV+EakGblPV/yciFwPfVdVmQ3lE5FrgWoBhw4ad+sknnzQ5bmtTJE5H1yOJVVf+zlWVQ++/T+VLRVT94x8EKirw9ulDz+nTySosJHXsSTbfl+n2OnM9kki+F0WaMg7XIgBygG2hCVR1j6o2zIH+CHBqyLE7VXWCqk7B6WBveHalDGeNFIAXcBbaakZVH1bVAlUt6NevXxTFNabziAhp+fkM/MmPGbXsLXJ+s4D0006j4umn2fKlL1E64yJ2/+5h/Fu2oMFgootrTEw6ulhDNP+UWgGMcmcL3gpcBlze5CQig1R1u7tZiDMKq6Gjvpeq7hGRfJxgsdhN9yJwPvAYcC7O6K1j1rG4HsmRRpKTyTz/fDLPP59AZSVVi16lsqiI8vvuo/y++5DUVJJzc0nJyyM5L4+UvBEk5+WRnJuLJ2QAhDHdVUebtj5V1TanTRWRLwLzAS/wmKreKSLzcIaUFYnIz3ECSD2wF7hOVT8UkVRglXuaKuBbqlrinrMX8BdgGFDtHlvdWjlsqV0D3e937v/sMw78+9/4SzdTu7kUf+lm6srKoOFvUgTfkCEk540gZUQeySPzGoONt3dvaxozcRdt01aLNRIR2Q9EijICpEVTCFVdCCwM2/fTkPe3ArdGyHcIZ+RWpHNWANOj+XxjurPkoUNJHjq0yb7goUP4P/kEf2kptaVOcKktLaVm+Qr00KHGdN6sLJJHjjwcZPJGkDJyJL4hQxCvt6svxRzjWgwkqprZlQUxxjjr0qeecAKpJ5zQZL8Gg9Rv305t6Wb8pZvcn6VUv/kWlc8935hOfD6Sc3OdJrKReSQ3BJkRI/Ckp3f15ZhjhC1obcwRQDwefEOGOA86nv25JscCFRXUbt6Mv3Qz/s2l1G4qpfbDD9m/ZAmEdOQnDR7k1l4a+mFGkpI3Am92tjWTmZhYIDHmCOft1Yv0k08m/eSTm+wP+v3UffoptZtKnQDjNpVVPv88wZDZoz2ZmU6tJc9tKstzazJDcxCfr6svxxyBLJAkSEUI45jXAAAf70lEQVRFBU8++STXX399u/POnz+fa6+9lvRu2FRx3nnn8atf/SrqdUFCvfjiixx//PGMGTMm6nN99NFHfOUrX2ncLi0tZd68edx4Y5uz+Bz1PMnJpBx3HCnHHddkv6pSv3On2w9zuKnswDvvUBk6Cs/nI3nYMKf2MsJtKsvLI3nECLwZGV18NaY7s0CSIA3rkXQ0kMyZM6dbBpJYvPjii8yYMaMxkETjhBNOoKSkBIBAIMCQIUOYNWtWvIp4VBARfAMH4hs4kB5nntnkWKC6uklHf0NT2f6lb0LIXGdJ/fs7o8hCm8pGjiSpf39rJjsGtRlIWhi9VQkUA/+rzvokR7R7lt/Dh3s/bDthO5zY50Runnhzi8dtPZLOX4/k9ddfZ+TIkQwfPrztX5CJyJuRQVp+Pmn5TZ/z1bo6/J+VNQYWf2kptZtLqSwqIujOEQfgSU93ai3hTWXDhiHJyV19OaaLRFMjuRfnifQncYb+XgYMxJke/jHgvHgV7mh29913s3btWkpKSli8eDHPPfccy5cvR1UpLCxk2bJllJeXM3jwYP7xj38AziSLWVlZ3HvvvSxdurTVKUYa1iO55557mDVrVuN6JOvWreOKK66gsLCwcT2S1NRUNmzYwOzZsykuLmbWrFk8//zzLFiwgEWLFkW9HsmaNWs45ZRTgKbrkfTo0YN77rmHe++9t3H24ob1SP70pz9x44038ve//53CwkJmzJjBpZde2nj+hvVIFi5cyO23385rr73Gtm3buPrqq1m4sMnIcp5++mlmz54d0+/FRCY+Hyl5I0jJG0Hm5z/fuF9VqS8vP1x7Kd2Mf9MmalYUU1X08uETeL3OcOeG2ktIU5ktEnbkiyaQTFPV00O2HxaRd1V1noj8MF4F60qt1Ry6wuLFi1m8eDEnu52l1dXVbNiwgbPPPpubbrqJm2++mRkzZrRrdcDk5OTGyQvHjRtHSkoKPp+PcePGNT5ZXldXx9y5cykpKcHr9TaZnffBBx9k7NixTJo0qdUv52XLljUuiJWfn0+++y/Zd999l3Xr1nHWWWcB4Pf7G2f5BRrPOXv2bL773e+2eP6LL74YgFNPPbWx3IMHD24WRPx+P0VFRfz85z9v896YziMi+Pr3x9e/Pz0mnd7kWPDAAWq3bGnaVFZayoF//rPJBJbe7GxSRow4/MDlCCfYJA0ahLSyfIHpPqIJJEER+TLwnLt9acix9j8Wb5qx9UhiW48E4JVXXuGUU05hwIABLaYxXcvTowdpJ51E2kknNdmv9fXUbd0a8sDlJvylm6l6ZRHBysrGdJKWRvKI3CYPXCaPyCM5dzge9/8L0z1EE0i+CtwP/Mbd/jcwR0TSgLkt5jKtCl+P5Cc/+Qlf/epXycjIYOvWrfh8Purr6+nTpw9z5swhIyODP/7xj03yxjp7bmVlJTk5OXg8Hh5//PGI65H86U9/4t577+Wmm26KeI6G9UgmT57cbD2Sb3/722zcuJHjjjuOmpoaysrKOP744wFnHZJbbrkl5vVIGjz11FPWrHWEkKQkkocPJ3n4cAhZCkFVCezbh3/T4QcuazeXcrCkhKqFC5tOHZOT07wfJi+PpN69E3RVx7Y2A4nbmX5RC4ff7tziHDtC1yO58MILG9cjAaej/M9//jMbN27k+9//Ph6PB5/Px0MPPQQcXo9k0KBBra7b3pbrr7+eSy65hGeffZbJkydHXI9kwoQJnHbaaUyfPj3iPFXXXXcdV111Ffn5+UyYMCHieiQNk0recccdjYGkYT2SYDDYWGu57LLLuOaaa3jggQd47rnnmn1Wg/A+kpqaGpYsWcLvfve7Dt8Lk3giQlKfPiT16UP6aac1ORY8ePDw1DGNz8Vspubd99CQSUu9vXs3e+AyOS8P3+DBNnVMHLU5aaOI5AAPAmfhNGW9DXxHVcviX7zOYZM2di/HwnokpmtoMEjdtm1hc5M5TWWBvXsb00lKijt1zIimQ5ZHjMCTFtXUgcekmCdtDPEHnBFbX3K357j7pnS8eMYYEzvxeEjOySE5J4eMc85pcqx+3z78m7c0mZvs0Lp17H91cZOpY3yDB0ccsuzt29eeiYlSNIGkn6r+IWT7jyJijw13E7YeiTGRJfXuTVLv3qSfEjZ1TG2t20y2uclzMRUrV6Ihz0F5srKc0WQha8Sk5OXhy8lBkuxZ7lDR3I3dIjIHaBh+MxvYE78imfZ47733uuRzpk6dytSpU7vks4yJJ09KCqnHH0+q21/XQINB6nfubPLApb90M9X/XEbl3/52OKHPR/LwYc3mJkvJG4HH7Wc81kQTSL4O/Bq4D6eP5F/AVfEslDHGdDXxePANGoRv0CD43FlNjgWqqvBv3txkbrLaDRvY//rr4I52BEgaOLDxgcvQIctJ/fsd1c1k0Yza+hRnFcNGbtPW/HgVyhhjuhNvz56kjR9P2vjxTfar34//s8+aPHBZW1pK5YsvEnSnHgLnmZqGprHD/THu1DFHwQzLHW3o+x5RBBIRmYbzDIoXeFRV7w47fiXwS5w13QF+raqPusfu4fBKiD9T1WfC8j4IXKWqNg2pMSYhJDmZlJEjSRk5ssl+VaV+V7nbB7OpsT/mwHvvUfnSS4cTJiU1nTomZMiyN/PIWVuwo4GkzTqaiHiBBTiju8qAFSJSpKrrwpI+o6pzw/JOB04BJgApwFsi8oqqVrnHC4BeHSy7McbElYjgG9Af34D+9Jg0qcmxQPUB/Js3H14jZpPTH1O9bBmETh3TL/twP0zDkOWReSQNHNjtmsk6GkiimRplIrCxYXZgEXkamAmEB5JIxgBvqWo9UC8iq4FpwF/dAPVL4HLgiJ0v3NYjaa4j65EA3H///TzyyCOoKtdcc42tRWK6NW9GD9LGjSVt3Ngm+7W+nrqyMreZrLRxyHLVPxYSrKpqTCfp6aREWE45OTcXT4JmWG4xkLQwfTw4tZFonuAZAnwWsl0GnB4h3SUicg7wMfBdVf0MWA3cJiL3AunAZA4HoLlAkapu725RuT1sPZLmOrIeydq1a3nkkUdYvnx540SV06dPZ9SoUXEsqTGdT5KSnIcmc3Ph/PMb96sqgT17mj1weXDVKqr+/vfDJ/B48A3NabaccuqY0XhC5tGLhxYDiarG2kAX6Vs+PDC9DDylqrUi8i3gceB8VV0sIqfhjBArx5nfq15EBuM8GHlemx8uci1wLcCwYcNaTbvjrruoXd+565GkjD6RgT9seXJkW4+kc9YjWb9+PZMmTWoMqueeey4vvPACP/jBD9rx2zKm+xIRkrKzScrOpoc7BVGDYE0N/i1bmsxN5t9UyoF//Qv1+wHIe7mIlDj/wyqeT9WUAUNDtnNw1jVppKqhz6M8AtwTcuxO4E4AEXkS2ACcDBwHbHRrI+kislFVm64l6uR/GHgYnClSOuF6OpWtR9I565GMHTuWH/3oR+zZs4e0tDQWLlzYoWY1Y45EnvR0UseMITWsFq+BQOPUMcldsNBbPAPJCmCUiIzAGZV1GU6/RiMRGaSq293NQmC9u98L9FLVPSKSD+QDi90+k4Eh+asjBZH2aq3m0BVsPZKOr0cyevRobr75ZqZMmUJGRgbjx48nyZ46Nsc4aVhIbOjQthN3grj9xalqvYjMBV7FGf77mKp+ICLzgGJVLQJuEJFCoB7YC1zpZvcB/3RrHVXAHDeIHJVsPZLY1iP5xje+wTe+8Q0AfvjDH5KTk9Pi+YwxnS+uy4+p6kJVPV5VR7pNVajqT90ggqreqqonqep4VZ2sqh+6+w+p6hj3NUlVS1o4/xH7DEn4eiSPPfYY1e7a11u3bmXXrl1s27aN9PR05syZw0033cSqVaua5Y1FZWUlgwYNwuPx8MQTT0Rcj2T06NHce++9LZ6jYT0SoNl6JO+88w4bN24EnKneQ2s8zzzzTOPPWNcj2bVrFwCffvopf/vb32xdEmO6mLUBJIitR9J565Fccskl7NmzB5/Px4IFC+htixsZ06XaXI/kaGDrkXQvth6JMUeGaNcjiWvTljHGmKOfNW0d4Ww9EmNMoh3TgURVu92cNe1l65FE51howjUmUY7Zpq3U1FT27NljXzDHAFVlz549TYY3G2M6zzFbI8nJyaGsrIzy8vJEF8V0gdTUVHu+xJg4OWYDic/nY8SIEYkuhjHGHPGO2aYtY4wxncMCiTHGmJhYIDHGGBMTCyTGGGNiYoHEGGNMTCyQGGOMiYkFEmOMMTGxQGKMMSYmFkiMMcbExAKJMcaYmMQ1kIjINBH5SEQ2isgtEY5fKSLlIlLivq4OOXaPiKx1X18J2f8X95xrReQxEfHF8xqMMca0Lm6BRES8wALgQmAMMFtExkRI+oyqTnBfj7p5pwOnABOA04Hvi0hPN/1fgBOBcUAacHWEcxpjjOki8ayRTAQ2qmqpqvqBp4GZUeYdA7ylqvWqegBYDUwDUNWF6gKWAzalqzHGJFA8A8kQ4LOQ7TJ3X7hLRGSNiDwnIkPdfauBC0UkXUSygcnA0NBMbpPW14BFnV90Y4wx0YpnIIm09GD4KlIvA7mqmg+8BjwOoKqLgYXAv4CngH8D9WF5fwMsU9V/RvxwkWtFpFhEim3NEWOMiZ94BpIymtYicoBtoQlUdY+qNiw4/ghwasixO91+kyk4QWlDwzERuQ3oB3yvpQ9X1YdVtUBVC/r16xfzxRhjjIksnoFkBTBKREaISDJwGVAUmkBEBoVsFgLr3f1eEenrvs8H8oHF7vbVwFRgtqoG41h+Y4wxUYjbComqWi8ic4FXAS/wmKp+ICLzgGJVLQJuEJFCnGarvcCVbnYf8E8RAagC5qhqQ9PWb4FPgH+7x/+mqvPidR3GGGNaJ87gp6NbQUGBFhcXJ7oYxhhzRBGRlapa0FY6e7LdGGNMTCyQGGOMiYkFEmOMMTGxQGKMMSYmFkiMMcbExAKJMcaYmFggMcYYExMLJMYYY2JigcQYY0xMLJAYY4yJiQUSY4wxMbFAYowxJiZxm/33qFDyJOzZBJ4k8HjdVxKIN2Rf6E/3vYRthx5vMW94viTweJpui9fZZ4wx3YgFktasK4INi0EDiS5JCIkciFoNXmEBKWJwayHAtRkUow2arQXbsKDZ3kAtkRbjNMZ0FQskrbn8aeenKgQDTkAJ1ruvgPtytzVsu/F9IOR4w7FgyPt60LDt0HO39pmxlCngj+283WlNMQkPQi0Et9aCYlQ1yfDjUeRrFsgjBWpvyHmi2e9pHnAbP8Mbtt+CrIk/CyTREAFvEs7tSkl0abqHYDAk2ARaCECRgmaAZkE2mqAZMSiGH48mKIaVqeG8gToIHmylTFGct1uSCAHGExZswoNWtGlaCmCe5rVK8XRi8IzHOUOux4Jvu1kgMR3j8QAe8PoSXZLuo1mAihAsIwahsKDc+D7YPIhqMCzIBULyhO6PJW+k/W7e+toIaerD0ofvD9Ls2rozCQ08nRxUExGQ886D1J5xvWUWSIzpLB4PeJKB5ESXpPuLGDxjDE6tnTMhATkI9f525G3lmmNpSv72iiM7kIjINOB+nDXbH1XVu8OOXwn8Etjq7vq1qj7qHrsHmO7u/5mqPuPuHwE8DfQBVgFfU1V/PK/DGNPJrEbbPqrtD54N+3sNi3vx4hZIRMQLLACmAGXAChEpUtV1YUmfUdW5YXmnA6cAE3A6Jd4SkVdUtQq4B7hPVZ8Wkd8C3wAeitd1GGNMwokcbrLqhuL5UMJEYKOqlro1hqeBmVHmHQO8par1qnoAWA1MExEBzgeec9M9DvxXJ5fbGGNMO8QzkAwBPgvZLnP3hbtERNaIyHMiMtTdtxq4UETSRSQbmAwMBfoCFara0FvX0jmNMcZ0kXgGkkhj6DRs+2UgV1Xzgddwahio6mJgIfAv4Cng30B9lOd0PlzkWhEpFpHi8vLyjl2BMcaYNsWzs70MpxbRIAfYFppAVfeEbD6C0//RcOxO4E4AEXkS2ADsBnqJSJJbK2l2zpD8DwMPAxQUFEQMNm35v3/9H+v2rGNA+gD6p/dnQA/nZ//0/gxIH8CA9AFkJGd05NTGGHPUiGcgWQGMckdZbQUuAy4PTSAig1R1u7tZCKx393uBXqq6R0TygXxgsaqqiCwFLsXpc7kCeCleFzCs5zB21exi+4HtrC5fzb7afc3SpCelHw4sIYGmf3p/BqYPpH96f/qk9sHbTTvJjDEmVqLaoX+sR3dykS8C83GG/z6mqneKyDygWFWLROTnOAGkHtgLXKeqH4pIKs7QXoAq4FuqWuKeM4/Dw3//A8xR1drWylFQUKDFxcUxX09toJZdNbsaXzsP7GRnzc7D2zU7Ka8pp16bPnDlFS/ZadkM6DGgsXbTEHxCf6YmpcZcRmOM6SwislJVC9pMF89A0l10ViCJRlCD7D20tzHQNASY0J+7anZRXVfdLG9WSlazprPw7ayULMSmcDDGdIFoA4k92d7JPOIhOy2b7LRsxvQd02K6A3UHmtZmwmo3H+39iD0H96BhYwlSvCn0S+vX2GcTKdhkp2fj89iDXsaYrmGBJEF6+HqQl5VHXlZei2nqgnXsObiHnTU7G2s3DTWbnTU7Wbt7LW/UvEFtoGnLniD0Se3T2GcT3oTW8NMGChhjOoMFkm7M5/ExsMdABvYYCP0ip1FVqvxV7Diwo2n/jRtstlVvo2RXCRW1Fc3y9vD1aL0prccA+qT2wSO2mJYxpmUWSI5wIkJWShZZKVmc0OeEFtMdqj9EeU15s/6ahp/Ldyxnd83uZgMFkiSJ7PTsZsGmyc8e/Unx2vT6xhyrLJAcI1KTUhnacyhDew5tMU3DQIFIo9F21uxkY8VG3tn6DjX1Nc3y9krp1WLtpn96fwb2GEjP5J42UMCYo5AFEtModKDASZzUYrpqf3XEWk3Dz/V71rP30N6IAwVabEpzBw70TetrAwWMOcJYIDHtlpGcQUZyBnm9Wh8osLtmd2Ntpknt5sBO1pSvYVfNLvzBpisACELftL4R+2tCt3v4esT7Mo0xUbJAYuLC5/ExKGMQgzIGtZhGVamsrYwcbGp2UlZdxqpdq6isrWyWN8OX0aTprEkNp4ezbQMFjOkaFkhMwogIvVJ70Su1V6sDBQ7WH4w4UKDh+Zv3tr/H7oO7CWigSb4kT9LhZ27CBwmE1HBsoIAxsbFAYrq9tKQ0hvUcxrCeLa/0FggGnIEC4bUb9/mbj/d9zD+3/pOD9Qeb5e2V0qvNpjQbKGBMyyyQmKOC1+OlX3o/+qX3YyxjI6ZRVarrwgYKhD3o+cGeD9h7aG+zvKne1KZNaRFmFchOyybJY39S5thj/9ebY4aIkJmcSWZyJiN7jWwxXV2gjvKD5eyq2cWOmh3sOtD0Qc/V5avZ9cku6oJ1TfJ5xEPf1L4Rm9BCm9bSfenxvlRjupQFEmPC+Lw+BmcMZnDG4BbTqCr7avc1G43WsP3p/k9ZsXMF+/37m+XN8GU0BpY+aX1I9aaS4k0hxZtCsje58X2T7aSm+1K9qY3HGrZTvCkkeZKsCc50OQskxnSAiDOfWZ/UPpzY58QW0x2sP9gYXCJNY/Pp/k/xB/zUBmrxB/wcChyKrVzI4cDjaR6IwgNVa4GpSdoW8jdsp3pTLYgdwyyQGBNHaUlpDO85nOE9h0eVXlWpC9ZRG6htElyaBJv6w9uhr/B9LaWpqq1qFrwatsMfIm2PhiAWMTAlpbYYxFrbjpQ/UqDzeXwWxBLIAokx3YiIkOxNJtmbTCaZXfrZqkp9sJ5DgUOtB6b62tYDXaDlQLffv78xvz/gpzZ4+HydFcQ6Wqtqs5bWQjC0IGaBxBjjEhF8Xh8+ry9hQaw2UNt2YKpvXw2sIf/+g/ub1uiCh2t4sQQxoP21Kk8HamkR+sm6SxCzQGKMSbjQIJZB166TExrEogpMIU2L/mB0TY376/Y3rdF1chCLWANzX/POmtfqwJHOENdAIiLTgPtx1mx/VFXvDjt+JfBLYKu769eq+qh77BfAdMADLAG+o6oqIrOBHwIKbMNZs313PK/DGHP06k5BrK2mwSY1smB0TY1dMU1Q3AKJiHiBBcAUoAxYISJFqrouLOkzqjo3LO+ZwFlAvrvrbeBcEXkbJzCNUdXdbrCZC/xfvK7DGGPiJZFBrDPFM1RNBDaqaqmq+oGngZlR5lUgFUgGUgAfsBMQ99VDnIbBnji1EmOMMQkSz0AyBPgsZLvM3RfuEhFZIyLPichQAFX9N7AU2O6+XlXV9apaB1wHvI8TQMYAv4/04SJyrYgUi0hxeXl5p12UMcaYpuIZSCINJQjvVXoZyFXVfOA14HEAETkOGA3k4ASf80XkHBHx4QSSk4HBwBrg1kgfrqoPq2qBqhb069fCgufGGGNiFs9AUgaEruuaQ1gzlKruUdVad/MR4FT3/SzgXVWtVtVq4BVgEjDBzbdJVRX4K3Bm/C7BGGNMW+IZSFYAo0RkhIgkA5cBRaEJRCR01aNCYL37/lOczvUktxZyrntsKzBGRBqqGFNC8hhjjEmAuI3aUtV6EZkLvIoz/PcxVf1AROYBxapaBNwgIoVAPbAXuNLN/hxwPk5fiAKLVPVlABG5HVgmInXAJyF5jDHGJIA4LURHt4KCAi0uLk50MYwx5ogiIitVtaCtdLagtTHGmJgcEzUSESnHaQbriGygOz45b+VqHytX+1i52udoLddwVW1z2OsxEUhiISLF0VTtupqVq32sXO1j5WqfY71c1rRljDEmJhZIjDHGxMQCSdseTnQBWmDlah8rV/tYudrnmC6X9ZEYY4yJidVIjDHGxMQCCSAij4nILhFZ28JxEZEHRGSjO1PxKd2kXOeJSKWIlLivn3ZRuYaKyFIRWS8iH4jIdyKk6fJ7FmW5uvyeiUiqiCwXkdVuuW6PkCZFRJ5x79d7IpLbTcp1pYiUh9yvq+NdrpDP9orIf0Tk7xGOdfn9irJcCblfIrJFRN53P7PZ09dx/3tU1WP+BZwDnAKsbeH4F3EmjhScySPf6yblOg/4ewLu1yDgFPd9JvAxzmJjCb1nUZary++Zew8y3Pc+4D1gUlia64Hfuu8vw1nwrTuU60qclUu79P8x97O/BzwZ6feViPsVZbkScr+ALUB2K8fj+vdoNRJAVZfhzPXVkpnAn9TxLtArbMLJRJUrIVR1u6quct/vx5k4M3ytmS6/Z1GWq8u596Da3fS5r/DOyZm4yyjgzDX3eXfxtkSXKyFEJAdnqe1HW0jS5fcrynJ1V3H9e7RAEp1oF+lKhDPcpolXROSkrv5wt0nhZJx/zYZK6D1rpVyQgHvmNoeUALuAJara4v1S1XqgEujbDcoFERaf6wLzgR8AwRaOJ+R+RVEuSMz9UmCxiKwUkWsjHI/r36MFkuhEs0hXIqzCmcJgPPAg8GJXfriIZADPAzeqalX44QhZuuSetVGuhNwzVQ2o6gScdXkmisjYsCQJuV9RlCvi4nPxJCIzgF2qurK1ZBH2xfV+RVmuLr9frrNU9RTgQuDbInJO2PG43i8LJNFpc5GuRFDVqoamCVVdCPhEJLsrPlucdWKeB/6iqn+LkCQh96ytciXynrmfWQG8CUwLO9R4v0QkCciiC5s1WyqXtrz4XDydBRSKyBbgaZwVUv8cliYR96vNciXofqGq29yfu4AXgIlhSeL692iBJDpFwH+7Ix8mAZWquj3RhRKRgQ3twiIyEef3uacLPleA3wPrVfXeFpJ1+T2LplyJuGci0k9Eernv04AvAB+GJSsCrnDfXwq8oW4vaSLLJS0vPhc3qnqrquaoai5OR/obqjonLFmX369oypWI+yUiPUQks+E9cAEQPtIzrn+PcVvY6kgiIk/hjObJFpEy4DacjkdU9bfAQpxRDxuBGuCqblKuS4HrRKQeOAhcFu8/JtdZwNeA9932dYAfAsNCypaIexZNuRJxzwYBj4uIFydw/VVV/y5NF3n7PfCEiGzE+Zf1ZXEuU7TlamnxuS7XDe5XNOVKxP0aALzg/vsoCXhSVReJyLega/4e7cl2Y4wxMbGmLWOMMTGxQGKMMSYmFkiMMcbExAKJMcaYmFggMcYYExMLJMYYY2JigcSYbsKdCrxDT9m705cP7oxzGdNeFkiMOTpcCQxuK5Ex8WCBxJgwIpIrIh+KyKMislZE/iIiXxCRd0Rkg4hMdF//EmeBo3+JyAlu3u+JyGPu+3Fu/vQWPqeviCx2z/E7QibWE5E54iw6VSIiv3OfPkdEqkXk/4nIKhF53Z3m5FKgAPiLmz7NPc3/uOneF5ET43nPzLHNAokxkR0H3A/kAycClwOfA27CmXblQ+AcVT0Z+Clwl5tvPnCciMwC/gB8U1VrWviM24C33XMU4U7lIiKjga/gzOg6AQgAX3Xz9ABWuTO9vgXcpqrPAcXAV1V1gqoedNPudtM95JbbmLiwubaMiWyzqr4PICIfAK+rqorI+0Auzmyzj4vIKJzpuBvmQAuKyJXAGuB3qvpOK59xDnCxm+8fIrLP3f95nFljV7jzJ6XhrBcCzjoYz7jv/wxEmnm5QcOxlQ2fY0w8WCAxJrLakPfBkO0gzt/Nz4ClqjpLnEW03gxJPwqoJro+i0iT3QnwuKre2sH8DRrKHMD+1k0cWdOWMR2TBWx131/ZsFNEsnCaxM4B+rr9Fy1ZhttkJSIXAr3d/a8Dl4pIf/dYHxEZ7h7z4MxgDE5z29vu+/0469Qb0+UskBjTMb8Afi4i7wDekP33Ab9R1Y+BbwB3NwSECG4HzhGRVThrSHwKoKrrgB/jLJ26BliCM+U7wAHgJBFZCZwPzHP3/xH4bVhnuzFdwqaRN+YIIiLVqpqR6HIYE8pqJMYYY2JiNRJj4kxErgK+E7b7HVX9diLKY0xns0BijDEmJta0ZYwxJiYWSIwxxsTEAokxxpiYWCAxxhgTEwskxhhjYvL/AVB0fV7oR9+aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7d1918bc90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch2_1.best_score_, gsearch2_1.best_params_))\n",
    "test_means = gsearch2_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch2_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch2_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch2_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch2_1.cv_results_).to_csv('my_preds_maxdepth_min_child_weights_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(max_depth), len(min_child_weight))\n",
    "train_scores = np.array(train_means).reshape(len(max_depth), len(min_child_weight))\n",
    "\n",
    "for i, value in enumerate(max_depth):\n",
    "    pyplot.plot(min_child_weight, -test_scores[i], label= 'test_max_depth:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'max_depth' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig('max_depth_vs_min_child_weght_1.png' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_depth': [6, 7, 8], 'min_child_weight': [4, 5, 6]}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_depth = [6,7,8]\n",
    "min_child_weight = [4,5,6]\n",
    "param_test2_2 = dict(max_depth=max_depth, min_child_weight=min_child_weight)\n",
    "param_test2_2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 在粗调的结果下进行精细调整"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58922, std: 0.00388, params: {'max_depth': 6, 'min_child_weight': 4},\n",
       "  mean: -0.58874, std: 0.00361, params: {'max_depth': 6, 'min_child_weight': 5},\n",
       "  mean: -0.58835, std: 0.00380, params: {'max_depth': 6, 'min_child_weight': 6},\n",
       "  mean: -0.58772, std: 0.00407, params: {'max_depth': 7, 'min_child_weight': 4},\n",
       "  mean: -0.58825, std: 0.00355, params: {'max_depth': 7, 'min_child_weight': 5},\n",
       "  mean: -0.58817, std: 0.00367, params: {'max_depth': 7, 'min_child_weight': 6},\n",
       "  mean: -0.59144, std: 0.00438, params: {'max_depth': 8, 'min_child_weight': 4},\n",
       "  mean: -0.59261, std: 0.00370, params: {'max_depth': 8, 'min_child_weight': 5},\n",
       "  mean: -0.59095, std: 0.00373, params: {'max_depth': 8, 'min_child_weight': 6}],\n",
       " {'max_depth': 7, 'min_child_weight': 4},\n",
       " -0.5877153366935085)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb2_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=205,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch2_2 = GridSearchCV(xgb2_2, param_grid = param_test2_2, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch2_2.fit(X_train , y_train)\n",
    "\n",
    "gsearch2_2.grid_scores_, gsearch2_2.best_params_,     gsearch2_2.best_score_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "精细调整后结果变为max_depth=7,min_child_weight=4;"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 对正则参数调优"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'reg_alpha': [0.1, 1, 1.5, 2], 'reg_lambda': [0.1, 0.5, 1, 2]}"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_alpha = [ 1, 1.5, 2, 2.5]    #default = 0, 测试0.1,1，1.5，2\n",
    "reg_lambda = [0.1, 0.5, 1, 2]      #default = 1，测试0.1， 0.5， 1，2\n",
    "\n",
    "param_test3_1 = dict(reg_alpha=reg_alpha, reg_lambda=reg_lambda)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58936, std: 0.00490, params: {'reg_alpha': 0.1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58867, std: 0.00382, params: {'reg_alpha': 0.1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58922, std: 0.00377, params: {'reg_alpha': 0.1, 'reg_lambda': 1},\n",
       "  mean: -0.58861, std: 0.00428, params: {'reg_alpha': 0.1, 'reg_lambda': 2},\n",
       "  mean: -0.58904, std: 0.00423, params: {'reg_alpha': 1, 'reg_lambda': 0.1},\n",
       "  mean: -0.58916, std: 0.00388, params: {'reg_alpha': 1, 'reg_lambda': 0.5},\n",
       "  mean: -0.58851, std: 0.00328, params: {'reg_alpha': 1, 'reg_lambda': 1},\n",
       "  mean: -0.58831, std: 0.00328, params: {'reg_alpha': 1, 'reg_lambda': 2},\n",
       "  mean: -0.58859, std: 0.00344, params: {'reg_alpha': 1.5, 'reg_lambda': 0.1},\n",
       "  mean: -0.58829, std: 0.00321, params: {'reg_alpha': 1.5, 'reg_lambda': 0.5},\n",
       "  mean: -0.58843, std: 0.00343, params: {'reg_alpha': 1.5, 'reg_lambda': 1},\n",
       "  mean: -0.58833, std: 0.00360, params: {'reg_alpha': 1.5, 'reg_lambda': 2},\n",
       "  mean: -0.58867, std: 0.00385, params: {'reg_alpha': 2, 'reg_lambda': 0.1},\n",
       "  mean: -0.58846, std: 0.00314, params: {'reg_alpha': 2, 'reg_lambda': 0.5},\n",
       "  mean: -0.58794, std: 0.00300, params: {'reg_alpha': 2, 'reg_lambda': 1},\n",
       "  mean: -0.58800, std: 0.00396, params: {'reg_alpha': 2, 'reg_lambda': 2}],\n",
       " {'reg_alpha': 2, 'reg_lambda': 1},\n",
       " -0.5879396977192718)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=205,  #第一轮参数调整得到的n_estimators最优值\n",
    "        max_depth=7,\n",
    "        min_child_weight=4, #第二轮参数调整得到的max_depth及min_child_weight\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def modelfit(alg, X_train, y_train, useTrainCV=True, cv_folds=None, early_stopping_rounds=100):\n",
    "    \n",
    "    if useTrainCV:\n",
    "        xgb_param = alg.get_xgb_params()\n",
    "        xgb_param['num_class'] = 3\n",
    "        \n",
    "        xgtrain = xgb.DMatrix(X_train, label = y_train)\n",
    "        \n",
    "        cvresult = xgb.cv(xgb_param, xgtrain, num_boost_round=alg.get_params()['n_estimators'], folds =cv_folds,\n",
    "                         metrics='mlogloss', early_stopping_rounds=early_stopping_rounds)\n",
    "        \n",
    "        n_estimators = cvresult.shape[0]\n",
    "        alg.set_params(n_estimators = n_estimators)\n",
    "        \n",
    "        print cvresult\n",
    "        \n",
    "        \n",
    "        # plot\n",
    "        test_means = cvresult['test-mlogloss-mean']\n",
    "        test_stds = cvresult['test-mlogloss-std'] \n",
    "        \n",
    "        train_means = cvresult['train-mlogloss-mean']\n",
    "        train_stds = cvresult['train-mlogloss-std'] \n",
    "\n",
    "        x_axis = range(0, n_estimators)\n",
    "        pyplot.errorbar(x_axis, test_means, yerr=test_stds ,label='Test')\n",
    "        pyplot.errorbar(x_axis, train_means, yerr=train_stds ,label='Train')\n",
    "        pyplot.title(\"XGBoost n_estimators vs Log Loss\")\n",
    "        pyplot.xlabel( 'n_estimators' )\n",
    "        pyplot.ylabel( 'Log Loss' )\n",
    "        pyplot.savefig( 'n_estimators_205.png' )\n",
    "    \n",
    "    #Fit the algorithm on the data\n",
    "    alg.fit(X_train, y_train, eval_metric='mlogloss')\n",
    "        \n",
    "    #Predict training set:\n",
    "    train_predprob = alg.predict_proba(X_train)\n",
    "    logloss = log_loss(y_train, train_predprob)\n",
    "\n",
    "        \n",
    "    #Print model report:\n",
    "    print (\"logloss of train :\" )\n",
    "    print logloss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     test-mlogloss-mean  test-mlogloss-std  train-mlogloss-mean  \\\n",
      "0              1.038769           0.000219             1.037598   \n",
      "1              0.989160           0.000502             0.986871   \n",
      "2              0.945887           0.000547             0.942708   \n",
      "3              0.909237           0.000366             0.905086   \n",
      "4              0.877379           0.000629             0.872094   \n",
      "5              0.849632           0.000601             0.843394   \n",
      "6              0.825107           0.000896             0.817715   \n",
      "7              0.803799           0.001033             0.795284   \n",
      "8              0.784924           0.000830             0.775169   \n",
      "9              0.768283           0.000741             0.757335   \n",
      "10             0.753681           0.000401             0.741801   \n",
      "11             0.740451           0.000260             0.727723   \n",
      "12             0.728766           0.000363             0.715180   \n",
      "13             0.718168           0.000339             0.703686   \n",
      "14             0.708808           0.000564             0.693500   \n",
      "15             0.700419           0.000601             0.684248   \n",
      "16             0.692857           0.000540             0.675819   \n",
      "17             0.686142           0.000529             0.668124   \n",
      "18             0.680004           0.000652             0.661309   \n",
      "19             0.674362           0.000744             0.654799   \n",
      "20             0.669533           0.000805             0.649206   \n",
      "21             0.664982           0.000952             0.643417   \n",
      "22             0.660876           0.001192             0.638559   \n",
      "23             0.657010           0.001315             0.633850   \n",
      "24             0.653663           0.001223             0.629690   \n",
      "25             0.650355           0.001479             0.625398   \n",
      "26             0.647384           0.001830             0.621569   \n",
      "27             0.644493           0.001981             0.617838   \n",
      "28             0.641844           0.001848             0.614361   \n",
      "29             0.639379           0.002040             0.611044   \n",
      "..                  ...                ...                  ...   \n",
      "175            0.592221           0.002562             0.461025   \n",
      "176            0.592260           0.002611             0.460262   \n",
      "177            0.592195           0.002697             0.459639   \n",
      "178            0.592128           0.002659             0.459101   \n",
      "179            0.592301           0.002583             0.458526   \n",
      "180            0.592296           0.002496             0.457937   \n",
      "181            0.592234           0.002570             0.457314   \n",
      "182            0.592138           0.002582             0.456723   \n",
      "183            0.592175           0.002469             0.456131   \n",
      "184            0.592157           0.002416             0.455583   \n",
      "185            0.592209           0.002561             0.454885   \n",
      "186            0.592177           0.002614             0.454339   \n",
      "187            0.592217           0.002587             0.453829   \n",
      "188            0.592257           0.002566             0.453214   \n",
      "189            0.592339           0.002583             0.452609   \n",
      "190            0.592405           0.002503             0.451918   \n",
      "191            0.592402           0.002558             0.451387   \n",
      "192            0.592401           0.002668             0.450862   \n",
      "193            0.592554           0.002736             0.450234   \n",
      "194            0.592608           0.002731             0.449720   \n",
      "195            0.592650           0.002715             0.449191   \n",
      "196            0.592666           0.002793             0.448574   \n",
      "197            0.592620           0.002807             0.448112   \n",
      "198            0.592654           0.002753             0.447528   \n",
      "199            0.592734           0.002758             0.446897   \n",
      "200            0.592787           0.002809             0.446304   \n",
      "201            0.592826           0.002852             0.445770   \n",
      "202            0.592787           0.002834             0.445149   \n",
      "203            0.592802           0.002893             0.444614   \n",
      "204            0.592756           0.002897             0.443995   \n",
      "\n",
      "     train-mlogloss-std  \n",
      "0              0.000205  \n",
      "1              0.000489  \n",
      "2              0.000470  \n",
      "3              0.000761  \n",
      "4              0.001060  \n",
      "5              0.000771  \n",
      "6              0.001049  \n",
      "7              0.001383  \n",
      "8              0.001480  \n",
      "9              0.001726  \n",
      "10             0.001860  \n",
      "11             0.001759  \n",
      "12             0.001674  \n",
      "13             0.001685  \n",
      "14             0.001648  \n",
      "15             0.001676  \n",
      "16             0.001918  \n",
      "17             0.002045  \n",
      "18             0.001910  \n",
      "19             0.002012  \n",
      "20             0.002167  \n",
      "21             0.002171  \n",
      "22             0.002016  \n",
      "23             0.002100  \n",
      "24             0.002318  \n",
      "25             0.002063  \n",
      "26             0.001747  \n",
      "27             0.001627  \n",
      "28             0.001665  \n",
      "29             0.001524  \n",
      "..                  ...  \n",
      "175            0.002114  \n",
      "176            0.002136  \n",
      "177            0.002201  \n",
      "178            0.002158  \n",
      "179            0.002145  \n",
      "180            0.002193  \n",
      "181            0.002194  \n",
      "182            0.002160  \n",
      "183            0.002045  \n",
      "184            0.002061  \n",
      "185            0.002097  \n",
      "186            0.002101  \n",
      "187            0.002113  \n",
      "188            0.002068  \n",
      "189            0.001926  \n",
      "190            0.001949  \n",
      "191            0.001963  \n",
      "192            0.001924  \n",
      "193            0.001999  \n",
      "194            0.002125  \n",
      "195            0.002205  \n",
      "196            0.002131  \n",
      "197            0.002047  \n",
      "198            0.002199  \n",
      "199            0.002222  \n",
      "200            0.002217  \n",
      "201            0.002215  \n",
      "202            0.002176  \n",
      "203            0.002186  \n",
      "204            0.002123  \n",
      "\n",
      "[205 rows x 4 columns]\n",
      "logloss of train :\n",
      "0.46979995204528124\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEXCAYAAABCjVgAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmcHHWd//HXp7vnnkkmx4TcF4QjICQQLjkVlEMNii4CgoIHi7t44LXg+nBZfl64XiiuiCwgIpc3KoJyKCBXEiAcCSEJ5L7vTObs7s/vj2/NpDOZSTpheqon/X4+HvVIdXV11WeqO/3u+lbVt8zdERERAUjEXYCIiBQPhYKIiHRSKIiISCeFgoiIdFIoiIhIJ4WCiIh0UiiI5DCzL5vZzXHXIRIXhUI/Y2a1ZrbIzC7MmVZnZkvM7AM506aZ2Z/MbKOZbTKzOWb2dTMbFD1/iZllzKwxGl43s08WuPZTzWxZIdexJ7qrx92/4e4fL9D6FpnZ6YVYdiH01fvV37bLvk6h0M+4eyNwGXC9mTVEk78NzHT3XwOY2VuBvwP/BA5293rgTCANHJGzuKfcvdbda4EPAN82s6l985fInjCzVNw1SIlwdw39cABuA+4CTgXWAyNynnsC+NFuXn8J8ESXac8CF+Y8ng68AmwihMwhOc8dEk3bFM0zPee5s4E5wFZgOfAFoAZoBrJAYzSM7OHv+jHw5+j1zwD757E9Dgb+BmwA5gHn7U09wDXAHdHrxgMOXAosBTYClwNHAy9Gf/sNOevZH3gkej/WAb8E6qPnfhGtqzla15fy2MaLgP+I1tUKpKLHy6O/ZR5wWjfb4jhgFZDMmfY+4MVo/BhgJrAFWA18r4dteiqwrIfnBgK3A2uBxcBXgET0XBL4brQN3gCuiLZjqodlLQJO7+G5TwALovf1vo7PDGDA94E1wOZoGx3W0/sd9//X/jTEXoCGvXzjYBCwMvqPd2nO9BogA5y6m9dfQk4oRF90m4ADo8cHAtuAdwBlwJei/5zl0eMFwJejx2+P/gMeFL12JXBSTp1HRuM9fsnk1HFb9AVwTPQl+Evg7t28pobwpX1p9Jojo+1y6J7WQ/ehcCNQCbwTaAF+DwwDRkVfSqdE8x8Qba8KoAF4DPhBzrJ3+PLb1TbOmf8FYAxQBRwU/Z0jc+rrNjCBhcA7ch7/CrgqGn8KuDgarwWO62EZPb5fhED4A1AX1fEa8LHoucsJX8qjo+39EHsRCtHnal30flYAPwIei547A5gF1BMC4hCiH0Y9vd8a8hvUfNRPuftGwi/MauC3OU8NIjQLruqYYGbfjo4rbDOzr+TMe1w0vZGwl/ALYH703AeBP7v739y9HfgO4YvprYRforXAt9y9zd0fAf4EXBC9th2YbGYD3H2juz+3h3/eb939WXdPE0Jhym7mfzewyN1vdfd0tL7fEJrEeqOe/+fuLe7+V8KX+F3uvsbdlwOPA1MB3H1BtL1a3X0t8D3glF0sd1fbuMMP3X2puzcTwr4i+lvK3H2Ruy/sYdl3Eb0fZlZH+PV8V872OMDMhrp7o7s/vScbw8ySUe1Xu/tWd19E2DO4OJrlPOB6d18WfU6/tSfLz/Eh4BZ3f87dW4GrgePNbHz0N9QR9hDN3ee6+8qcv+/NvN8lTaHQT5nZRYRfaA8B1+U8tZHQTDGiY4K7f8nDcYXfEX5Jd3ja3es9HFMYDhwKfCN6biShWaBjGVnCr9RR0XNLo2kdFkfPAbyf8CW02Mz+YWbH7+GftypnvIkQQLsyDjg2CrhNZraJ8IUyvJfqWZ0z3tzN41oAMxtmZneb2XIz2wLcAQzdxXJ3tY07LM15fgHwWcLezJpoXSN7WPadwLlmVgGcCzzn7h3r+hhhL+VVM5thZu/eRY3dGUrYQ1ycMy33/R+ZW3eX8T3Rdfs0EprmRkU/RG4gNDWuNrObzGxANOubfb9LmkKhHzKzYYT21E8A/wqcZ2YnA7j7NkI7/Ll7skx3X034df2eaNIKwpdtxzqN0IyxPHpujJnlfn7GRs/h7jPc/RxCE8vvgXs7VrMnNe2BpcA/ooDrGGrd/ZN9XM83o2Ue7u4DgIsITRsduq5vV9u429e4+53ufmL0OmfHHwS5880hfKGeBVxICImO5+a7+wWE7XEd8Gszq8n/z2Qd4df4uJxpne8/oflmdM5zY/Zg2bm6bp8aYAjbP2c/dPejCD9mDgS+GE3v6f2WPCgU+qcbgN+7+6PRLvOXgJ9FvwqJHn/UzK6KAgQzGw1M6GmBZjaEcDDylWjSvcC7zOw0MysDPk842PkkIXS2AV8yszIzO5UQJnebWbmZfcjMBkZNIlsIzR4QfmEPMbOBvbQdOvwJONDMLo7qKTOzo83skD6up45wEHmTmY0i+pLKsRqYmPN4V9t4J2Z2kJm9PXqfWwh7KZnu5o3cCXwaOJlwTKFjOReZWUO0Z7IpmtzjcsysMncg7IneC3w9Oh16HPA5wp5Rx9/1GTMbZWb1hIPju1PWZT2pqP5LzWxK9Dd/A3jG3RdF7++x0XbbFm2PzG7eb8lH3Ac1NOzZALyX8Auqvsv0h4Gv5zw+Frif8J9+E/Ay8HVgSPT8JYT/LB1n3qwhtDkPy1nG+wgHDDcD/yA6cBs9d2g0bXM0z/ui6eXAA4RmrC3ADODEnNfdQmgC2ETPZx99Lefxqezm4HQ030GEM5bWRst/hHAsYo/qofsDzamc+ZeRcxCf8EX4lZxtMivani8QvuSX5cx7DrAkWtcX8tjGi9jxwPThhGM/WwkH4//U3TbMmX8s4Qv8z12m3xG9342EHwHv7eH1p0Z/f9fhAMKxqzui7b0U+Crbzz5KEfZk1xPOPrqSsGdhPaxnUTfr+Fr03OWEg+Ydf+/oaPpphDOOGtl+plft7t5vDbsfLNrAIiIFYWZnATe6+7jdziyxU/ORiPQqM6sys7PNLBU1o/0X4SQH6Qe0pyD9gpmdBPylu+c8nD0lRcLMqglNYQcTjnv8GfiMu2+JtTDJi0JBREQ6qflIREQ69btOtoYOHerjx4+PuwwRkX5l1qxZ69y9YXfz9btQGD9+PDNnzoy7DBGRfsXMFu9+LjUfiYhIDoWCiIh0UiiIiEgnhYKIiHRSKIiISCeFgoiIdFIoiIhIp5IJhYVrG/ntrKWoWw8RkZ6VTCisfvB7vOu+qTQ2NcVdiohI0SqZUKisrqXC2tm4duXuZxYRKVElEwoVA8M93LeuXxFzJSIixatkQqF68AgAmjZoT0FEpCclEwp1Q0MotG1eFXMlIiLFq2RCYVDDaAAyW9fEXImISPEqmVBIVtbSTAW2bW3cpYiIFK2SCQWATYl6yprXxV2GiEjRKqlQaEwNprJtfdxliIgUrZIKhZbyIdSmN8VdhohI0SpYKJjZLWa2xsxe7uF5M7MfmtkCM3vRzI4sVC0d0lVDGJjdqK4uRER6UMg9hduAM3fx/FnApGi4DPhJAWsBIFvdwGC20NjcWuhViYj0SwULBXd/DNiwi1nOAW734Gmg3sxGFKoegOSAYSTN2bBudSFXIyLSb8V5TGEUsDTn8bJo2k7M7DIzm2lmM9eu3ftTSsvV1YWIyC7FGQrWzbRuG/vd/SZ3n+bu0xoaGvZ6hdX1IRTU1YWISPfiDIVlwJicx6OBgv6Erxs6EoDWTerqQkSkO3GGwn3Ah6OzkI4DNrt7QX/C13d2daFjCiIi3UkVasFmdhdwKjDUzJYB/wWUAbj7jcD9wNnAAqAJuLRQtXRIVtfT7kl86bOFXpWISL9UsFBw9wt287wD/16o9XfLjI3JwZRV1vTpakVE+ouSuqIZYGvZUKpb1SmeiEh3Si4Umiv3oz6tUBAR6U7JhUKmZj+G+kZa05m4SxERKTolFwpWN4IB1sTa9RvjLkVEpOiUXCiUDwoXTW9YvSTmSkREik/JhULN0HCtQuO6pbuZU0Sk9JRcKAwcFi6iblm/POZKRESKT8mFQl3HVc1b1P+RiEhXJRcKVllPC+UkG9X/kYhIVyUXCpixKTmEimb1fyQi0lXphQKwtXwYNW3r4i5DRKTolGQotFUNoz6zXvdqFhHpoiRDIVu7H8PYyMZtbXGXIiJSVEoyFMrqR1NtraxapYPNIiK5SjIUqhvGAbBx1cKYKxERKS4lGQqDRu4PwLY1i+ItRESkyJRkKNQOGw9AeoP6PxIRyVWSoWC1w2ilnOTWZXGXIiJSVEoyFDBjY2oYVU3q6kJEJFdphgKwrWo4g9p19pGISK6SDYX22tEM97VsaWmPuxQRkaJRsqFg9WNpsM2sWKs7sImIdCjZUKiKrlVYv+KNmCsRESkeJRsKA4dPBKBxtUJBRKRDQUPBzM40s3lmtsDMrurm+XFm9rCZvWhmfzez0YWsJ9eA4RMAaF2/uK9WKSJS9AoWCmaWBH4MnAVMBi4ws8ldZvsOcLu7Hw5cC3yzUPXsVN+AUWTdsCVP9dUqRUSKXiH3FI4BFrj76+7eBtwNnNNlnsnAw9H4o908XzipcjaWNVCVyvbZKkVEil0hQ2EUsDTn8bJoWq7ZwPuj8fcBdWY2pOuCzOwyM5tpZjPXrl3bawVurR7DkNYVZLK6r4KICBQ2FKybaV2/fb8AnGJmzwOnAMuB9E4vcr/J3ae5+7SGhoZeKzA9cDxjbDUrNzf32jJFRPqzQobCMmBMzuPRwIrcGdx9hbuf6+5Tgf+Mpm0uYE07KBs6kaG2hWWr1vTVKkVEilohQ2EGMMnMJphZOXA+cF/uDGY21Mw6argauKWA9eykbsQkADYsm9eXqxURKVoFCwV3TwNXAA8Cc4F73f0VM7vWzKZHs50KzDOz14D9gK8Xqp7u1I86EICmVQv6crUiIkUrVciFu/v9wP1dpn01Z/zXwK8LWcOuJIaEC9jYqAvYRESghK9oBqByIFsTA6hq1M12RESg1EMB2FI1mkEty8nqtFQREYVC+4BxjGE1yzfptFQRkZIPhfLNixhp61i4cn3cpYiIxK7kQ2Hg2z5N0pw1S16NuxQRkdiVfCjUjDoEgOYVc2OuREQkfiUfCgwJF7AlN+haBRERhUJFLZvLhjGg8Q3cdQaSiJQ2hQKwbcBExvoy1mxtjbsUEZFYKRQAGzqJ/W0l81dtjbsUEZFYKRSA2tGHUmfNLF+6MO5SRERipVAAaqMzkLYumxNzJSIi8VIoADb0IAAya1+LuRIRkXgpFADqhtOSrKVuywLdmlNESppCAcCMxgGTOIAlvLFuW9zViIjERqEQseGHcrAtZe6KPrsbqIhI0VEoRAaMO4IB1sSyxbqyWURKl0IhUjbiMABalr8YcyUiIvFRKHQYFk5LLV+v3lJFpHQpFDpUDaKxYj9GtC1irbq7EJESpVDIkR56CAfbUl5ctinuUkREYqFQyFGzZSH723JeWrw27lJERGKhUMhRdsZ/U2Fp1i/SwWYRKU0KhVwjpgCQXDVb91YQkZJU0FAwszPNbJ6ZLTCzq7p5fqyZPWpmz5vZi2Z2diHr2a3BE2lL1TEpPZ9F65tiLUVEJA4FCwUzSwI/Bs4CJgMXmNnkLrN9BbjX3acC5wP/W6h68mJG+7DDOSzxBrOX6mCziJSeQu4pHAMscPfX3b0NuBs4p8s8DgyIxgcCKwpYT16qxh3FwbaEb/9ZxxVEpPTsNhTMbH8zq4jGTzWzT5tZfR7LHgUszXm8LJqW6xrgIjNbBtwPfKqHGi4zs5lmNnPt2sKeGZQYNZUKS3Nk5cqCrkdEpBjls6fwGyBjZgcA/wdMAO7M43XWzbSuR28vAG5z99HA2cAvzGynmtz9Jnef5u7TGhoa8lj1mzByKgADNr7M5qb2wq5LRKTI5BMKWXdPA+8DfuDuVwIj8njdMmBMzuPR7Nw89DHgXgB3fwqoBIbmsezCGTSe9opBHGELmbl4Q6yliIj0tXxCod3MLgA+AvwpmlaWx+tmAJPMbIKZlRMOJN/XZZ4lwGkAZnYIIRTivXLMjMSYozkqOZ9nFykURKS05BMKlwLHA1939zfMbAJwx+5eFO1dXAE8CMwlnGX0iplda2bTo9k+D3zCzGYDdwGXeBFcIJAceywH2HJeWbg47lJERPpUanczuPsc4NMAZjYIqHP3b+WzcHe/n3AAOXfaV7ss+4Q9KbhPjDkGgNSKWWxrPZ2ait1uJhGRfUI+Zx/93cwGmNlgYDZwq5l9r/ClxWjkkbglmJqYzzNvrI+7GhGRPpNP89FAd98CnAvc6u5HAacXtqyYVdTiww5jWmIBj89fF3c1IiJ9Jp9QSJnZCOA8th9o3ucltq3lyMQ8nnxtddyliIj0mXxC4VrCweKF7j7DzCYC8wtbVhE48+tU0UblupdYubk57mpERPrEbkPB3X/l7oe7+yejx6+7+/sLX1rMxp8EwHGJuWpCEpGSkc+B5tFm9jszW2Nmq83sN2Y2ui+Ki1XtMLzhYE4pf5VH5q6JuxoRkT6RT/PRrYSLzkYS+i76YzRtn2fjT+Qoe5Un56+kpT0TdzkiIgWXTyg0uPut7p6OhtuAAndAVCTGn0RFtpkD2ufz5EI1IYnIvi+fUFhnZheZWTIaLgJK4+T98ScCcGryRf42R2chici+L59Q+CjhdNRVwErgA4SuL/Z9NUNhxBTeVTuPv81ZQyYbew8cIiIFlc/ZR0vcfbq7N7j7MHd/L+FCttJw4BlMbJlLunEdM9RBnojs4/b2zmuf69Uqitmkd2JkOa38Ff44O/Ybw4mIFNTehkJ3N9DZN42cCtVD+ODAuTzw8irSmWzcFYmIFMzehkLpNK4nkmAJpjQ+xqZtzTz1emkcYxeR0tRjKJjZVjPb0s2wlXDNQul413cp91ZOrlzIb59bHnc1IiIF02MouHuduw/oZqhz99K6wcABp0Oqkk8MfZn7X1rJ5mbdu1lE9k1723xUWspr4IDTmdb8T1rTGR1wFpF9lkIhX4e8h/JtK5k+dBX3zlwadzUiIgWhUMjXgWcAxr+23c6LyzYze+mmuCsSEel1CoV8VQ2Cg87ikPLV1JUbP39yUdwViYj0uny6zu7uLKSlUXfaE/uiyKJx+HkkGlfx+QPX8KcXV7KusTXuikREelU+ewrfA75I6DZ7NPAF4GfA3cAthSutCB14JlQM4NzUP2nLZDnnhifirkhEpFflEwpnuvtP3X2ru29x95uAs939HmBQgesrLmVVkKpiwLzf8O6DB9DYmqGxNR13VSIivSafUMia2XlmloiG83KeK50rmzt88HbwDF8aM5fNze3c8fTiuCsSEek1+YTCh4CLgTXRcDFwkZlVAVfs6oVmdqaZzTOzBWZ2VTfPf9/MXoiG18ys+E/pGXMsDD2QsW/8ioFVKb7z4Dy2aW9BRPYR+XSd/bq7v8fdh0bDe9x9gbs3u3uPjepmlgR+DJwFTAYuMLPJXZZ9pbtPcfcpwI+A3765P6cPmMGRH4Zlz3L3e+tJZ52bH38j7qpERHpFPmcfjY7ONFpjZqvN7DdmNjqPZR8DLIhCpY1wYPqcXcx/AXBXfmXH7IgLAOOQv32EMw8dzk2PLWS9zkQSkX1APs1HtwL3ETrBGwX8MZq2O6OA3Et/l0XTdmJm44AJwCN5LDd+NUNhyoXQupn/OHUY29oynHX943FXJSLypuUTCg3ufqu7p6PhNqAhj9d1d8+Fng5Mnw/82t0z3S7I7DIzm2lmM9euXZvHqvvAcZ+E9iYmLP41Hz1hAmsbW3lp2ea4qxIReVPyCYV1ZnaRmSWj4SIgn5sKLAPG5DweDfTUk9z57KLpyN1vcvdp7j6toSGfPOoDw98ClQPh0W/w2beNZUhNBV/5w8u6CY+I9Gv5hMJHgfOAVcBK4APApXm8bgYwycwmmFk54Yv/vq4zmdlBhOsdnsq36KLxgVsh08aAV3/FV98zmdlLN/GTvy+MuyoRkb2Wz9lHS9x9urs3uPswd38vcG4er0sTTll9EJgL3Ovur5jZtWY2PWfWC4C73b3/XfOw/9th1FHwxPeYflgD048YyfUPz1dneSLSb9nefBeb2RJ3H1uAenZr2rRpPnPmzDhW3b15D8BdH4QhB7D5Y09z5vWPUVWW5E+fPpHq8tK6F5GIFC8zm+Xu03Y33972ktrdQeTSdOAZMPJIaG9hYFmG7553BG+s38bX/jw37spERPbY3oZC/2vqKRQzOP0a2LIMZvyMt+4/lMtOmsidzyzhN7OWxV2diMge6TEUeugye4uZbSVcsyAdJp4ClfXw0DXQtIEvnnEQx08cwtW/e4nnlmyMuzoRkbz1GAruXufuA7oZ6txdjeVdXXo/uMOjXyeVTHDDhVMZPqCSS255lldW6PoFEekfdOe13rLfoXD0x2HGzfDTUxhSW8EvP34sLe1Zzrnhn8xfvTXuCkVEdkuh0JvedjUkymDdfEi3MWZwNQ9eeTKDasq58OZnWLCmMe4KRUR2SaHQm6oGwb/cBu3b4PHvADBhaA13fvxY3J0P3PgksxZviLdGEZFdUCj0tkPeDTUN8I/rYMXzAEzar47ffvIEWtoyfODGp3jg5VUxFyki0j2FQiFcMQPqRsDvPgnp0KX22CHVPHn1aUwZU8/ld8zihG89on6SRKToKBQKoWoQTP8RrJ0LPziic/LgmnLu/PhxnDt1FMs3NTPl2r+xdENTjIWKiOxIoVAok94Bb/0UNK6EF+7snFxVnuR7H5zC9edPwQzOuv5xfvf8Mvpj108isu9RKBTSadeE7rV//2+dxxc6nDNlFH/5zElMHjGAK++ZzVFfe4g1W1riqVNEJKJQKKRkCj71HAwcDfdcDNvW7fD06EHV3HXZcYweVMXGpjaO/+YjnPI/j5LJaq9BROKhUCi0mqHwwV/AluVw/RHQsuPVzcmE8cR/vJ1HPn8qNRVJFq9v4i3XPMijr65Rk5KI9DmFQl8YORUuuAfatsH3D4O2nQ8uTxhaw+z/eic/umAqQ2rLufS2GRx+zV95auF6hYOI9Jm9up9CnIrufgp74oZjYN08OOB0OP8uSJV3O1tbOss9M5dy7R9foT3j1JQnGTGwkgc+ezKppHJcRPZcvvdTUCj0teduh/s+FXpVvfJlqKjrcdbmtgy/eW4ZX/vTHFrSWcqSxkdPnMD7jxzNgfv1/DoRka4UCsXs+TvgD/8O5bXwmdnhuMMuZLPOQ3NXc8+MpTz86hoAqsuTXHbyRM44dDgHD6/DTPc9EpGeKRSK3Y+PC01JgybAxb+DQePyetm6xlb+8MIK7n9pJbMWh3s1VKQSfPj4cbzt4GFMGzeY8pSamERkRwqF/mDJ03Db2WBJ+MSjMPywPXr5mq0tPDRnDdc98Cpbmts7b4c3qLqML515MKce1MCIgVW9X7eI9DsKhf5izVz46cmQzcB5t4cO9fZCY2uaJxes4yu/f5lNTe20Rf0qVZUlef9Rozhq3CCmjRvM6EFVamoSKUEKhf5k83K45yJY8RwMHAOfeRESe98E5O7MX9PIx38+g01N7TS2pum4Hq4sadRWpLj4+PFMHjGAySMGMGawgkJkX6dQ6G/aW+DPn4cX7oDKQXDFs1A7rFcWnck6r67awnOLN/KDh+azsamN3Iumk2ZUlydpSWcYXV/F/3vvWxhZX8nI+ioqy5K9UoOIxEuh0B+5w8z/gwe+DJUD4Jz/hQPfWZBVNbdlmLd6K1fe/Tzb2jI0tWVobE3vNF9Z0shknfrqcsqTCSpSCb7y7smMqq9iVH0VA6pS2ssQ6QeKIhTM7EzgeiAJ3Ozu3+pmnvOAawAHZrv7hbta5j4dCh1Wz4GbT4P2JqgdDv/2FFQPLvhqs1lnxeZmlm9s5urfvkRbOktrOsuGpjbKkkZrOkvXj0vCoCKVpC2TZUhNORWpEBzXfeAIRg+qoqG2gkRCoSESt9hDwcySwGvAO4BlwAzgAnefkzPPJOBe4O3uvtHMhrn7ml0ttyRCAUJz0g+nwtYVkEhB/bhw855EfM057s77f/IkrVFYLN3QRH11GW3pLJub2zGzbjvzq0glSCWNlvYsg6rLKEsmWL+tjdH1VaSSRlkiwU8uOorBteXUlCe15yFSAMUQCscD17j7GdHjqwHc/Zs583wbeM3db853uSUTCh1WvQy3ng2tm6GsBj70Kxh/QtxV9aixNc0FNz3VuZexYlMzdZVlpLNZtrakSSUStGd33uPIVZY0UokEbekMdVVllCWMTc3t7DegklTCSCaMq886hJqKFLUVKWoqktG/KaoVKiLdKoZQ+ABwprt/PHp8MXCsu1+RM8/vCXsTJxCamK5x9wd2tdySCwUIxxrm/AH++hXYvBSqh8K//iN0yd0PuTvb2jJsaGzjk3fM4vV1jYwYWEV7JsvqLS0MrC4nncmypSVNRSpBOuOdp9jmI2lG1p2KsgTJRNhDqatIkTDY0pJmcE05CYMN29oYNqCShBmrt7Qwqr6KL55xEJVlSSrLklSVJ6ksS1BVlqSqLElleZLKVJKypCl4pN8phlD4F+CMLqFwjLt/KmeePwHtwHnAaOBx4DB339RlWZcBlwGMHTv2qMWLFxek5qLX1gT/vB4e+3YIiqM/Bide2W/DYU+1tGfY0tzO1tY021rTNLamueYPr5BxJ5N1Vm5uYWhtOZmss35bG3WVZWSyTmNLO5VlSbIOLekMqURo5uqN21aUJY2EGe2ZLJVlScygpS1DTUWKRMJobElTX11GwoyNTW0MqakgYeHK9GEDKjFgzdZWRgys5PJT9ieVTFCWNMqSiWjYeTwZ7S0lLPybShiJhJE0I5EIoZhKJEgmrXPPKpVQkJW6YgiFfJqPbgSedvfboscPA1e5+4yelluSewpdbVoCj38XZv0c8HAw+uMPQf2YuCvrV9yd9ozTks7Q0pahuT1DS3uWz937Atmsk3Vn8YYmRg6sIushdBrqKshmnbWNrQDUV5fj7mxqaqe2MoU7NLa0U1UehVB7hlQyQTbrpLNOwkKex3nOX8LACHtTqWQIinTGKU8lMAu99HacitzSnqGqPIw3t2WoLk9hQFNbmuqKML6tLRMdC4LG1gy10fStrWkGVKYA2NoSjZuxpbmdgVVlAGxubqe+Ome8qhwMNjW1Mag69CK8scs40ONzueP10fimpraGsNm8AAASpklEQVSwXGBTc9fxsO6Nze0MyhnvmL6paXt9uxrP/XvyHXdgS3M7A6LpW5rbqcvZXnWVKbxjvCIFBvvVVfDglafsydvdqRhCIUVoGjoNWE440Hyhu7+SM8+ZhIPPHzGzocDzwBR3X9/TchUKOTYtgZvfAY2rAIOjLgl7Dnn2oyTx6QikdDZLe9ppz2Zpz/Qwns7Sntk+nsl6595R1p10JvybyULGnWzWueWJN3Acd1i9tZWG2grcnXWNrQyuqQDC3tSg6nLcw5do7hdVXRRwW1vT1FakAKexNXz5QwiC6vIkeAiIqvIwT1NbhqooUJrbM1SWJXGH1nSGilQSx2lNZ6lIJnCgLZOlLJkAh/ZMdoeQSkZnrWWyO44DOz1nFl6T+/p8xsuirujbO+robjzqS6w9ne3sV6wtk6U8uWfjFdF4ayZLRcdycpeZzlKRCtuuNZ3ZIZg7xkfWV/HQ5wobCqm9Wnoe3D1tZlcADxKOF9zi7q+Y2bXATHe/L3runWY2B8gAX9xVIEgX9WPhC/Ng01J44vsw8xaYdSscMh2O+zcYexyoyaAomRnlKaOcBHR/W4035SNvHd/7C5WSoIvX9iWbl8P/vTPsOWTTUF4DZ38XDn0flFXGXZ2IxCj25qNCUSjkoa0JXrwbHvwytDeH6xxqGuBDv97jnlhFZN+Qbyio4/19UXk1TPsofHklfPi+0JfS1pVw4wlw09tg5q3QsiXuKkWkCGlPoVQ0bYAX74WH/zt0n2EJOOJCOOJ8GPfWWK+UFpHCU/ORdM8dfnpKOO6wbR14BhJlMOVCmDwdxp8MqQIc+RSRWMV+9pEUKTO4/LEw3rYN5v8V5twX7hv93M/DHsNbzoND3gP7vx3KdOc2kVKiUChl5TXhzKRD3xc64Hv90RAQL90Ls+8KTUyTzwmnuE56J1TUxl2xiBSYmo9kZ5l2eOMxmHsfPP9LyLaHgDjwrNDEdOCZUFUfd5Uisgd0TEF6RzYDS54KexCzboVM6GaAsW+FA94OB5wOw494U7cPFZHCUyhI78tmw32kX/0zLHwYVs4O0+tGhL2Hg86GCSfrQjmRIqRQkMJrXAMLHoJ5fwlB4Rkoqw57D4dMh4mnQm1D3FWKCDr7SPpC7bBwKuuUCyHdCoseh1fvh+dvD8cjIITEUZfA+BNh7PF9cltREdl72lOQ3pfNwIoX4I2/h/s/tG4Fj26Ss99bQkCMPzFcNKeQEOkTaj6S4pFuheWzYNET0fB4TkgclhMSJygkRApEoSDFK90Ky58LAbH4iXD6a0dIlFXD1Iuj5qbjQhOViLxpCgXpP9Jt4aymRY/DkzdA65btIZGqDBfQjT4axhwDww6FpA6FiewphYL0X+k2WPE8LH0Glj0Lrz0QLqiDcBFdeR0c84kQEqOPVpOTSB509pH0X6lyGHtsGCB04rdpCSybEYLihTvh8e/kzF8JB78LRhwRhuGHKyhE9pL2FKR/atsW7U08Gw5iz/8bZFq3P5+sgAPfGa627giLuv3iq1ckZtpTkH1bec32s5Y6NG2AVS+GK607hrl/3P58sjy87uhPbA+KgaN1H2uRHAoF2XdUDw5XUU88dfu0li2w+uXtITHnD/DYt7c/n0jB+JNg5BQYeSSMnKqgkJKm5iMpPW1NsGZOzh7FC7DyRSD6v5AoC8EyIqfpqX6sgkL6NTUfifSkvBpGTwtDh/YWWP1KODV2+XMhLBb8bfvziVRoepp6MYyYEoJiyP66janscxQKIhB6dh19VBg6tDfD6jmwKtqjeOnX8NSP6dyjsEQ4JXb44TD8LeHq7GEHh/AQ6afUfCSyJzLtsO61qNkpOqi99JnQQ2yHVGXoKXbYZNjv0DAMnqi9ComVLl4T6SvZLGxaHJqf1syBZ34aTplNN2+fxxKhC4/J54SQ6AgMdeMhfaQoQsHMzgSuB5LAze7+rS7PXwL8D7A8mnSDu9+8q2UqFKTfaG+GtfO2h8Xzd4SwyLZvnydRFo5xTPnQ9rBoODhME+lFsYeCmSWB14B3AMuAGcAF7j4nZ55LgGnufkW+y1UoSL/XuBbWvBKOVzzxA2haG6Z39PcEkKoKwXDMZdvDYtAE3fZU9loxnH10DLDA3V+PCrobOAeYs8tXiezrahug9tRw2uvx/xamZTOw4Y3tYfHsz6B5I/z9m9tfZ4lwHUXusYqGQ6BmqE6XlV5TyFAYBSzNebwMOLab+d5vZicT9iqudPelXWcws8uAywDGjh1bgFJFYpZIwtADwjD5HHjb1WF62zZY82oUFq/AC3eFU2bJ2cNPpMJZUEMPDE1PDdG/A0YpLGSPFTIUuvs0dm2r+iNwl7u3mtnlwM+Bt+/0IvebgJsgNB/1dqEiRau8ZsdTZc+6LnQQ2Lg6hMTaebD2VXjld7DkaXb4L2ZJKKsKB7iP+2QUGAdB/Th1Py49KuQnYxkwJufxaGBF7gzuvj7n4c+A6wpYj8i+wQzqhofhgNPCtOk/DGGxbR2si4Ji7Wvh3yVPwcP/nbuA7WFx7OWw32QYehAMGgfJslj+JCkehQyFGcAkM5tAOLvofODC3BnMbIS7r4weTgfmFrAekX2bWXS8omHHjgIBmjfBuvnhGov180NgLHwYHv1a7gLCNRZlleFsqCEHbB/qhqspqkQULBTcPW1mVwAPEk5JvcXdXzGza4GZ7n4f8Gkzmw6kgQ3AJYWqR6SkVdXDmKPDkKu1EdbMDUGxfgHMui0c4H76f3c8G8oSYe/iwDNzwmL/8G/lwD79U6SwdPGaiOwsm4Uty0NQdAyz7wkX5KVbusxsUFEHh74Phk4KB7yHHKBjF0Um9usUCkWhIBKzdGs4fTY3MNbNh+UzIZvecd5UZdjDSFXBSZ8L3X0M2R8GjlG3H32sGK5TEJF9UaoidPw37OCdn2vasP3YxYaFsOF1mP9QaJK6/ws5M0bHL7LtUDMMTv58CIzB+4f7WSgwYqNQEJHeUz14x/trd3CHrau2B8X6hfD8L0JYNK6CP38+Z2YLB7Ur66O9jEp4zw9CYNSN0FXdBabmIxGJVzYbgmH9whAa//h2uA4jWR76j+p6eZMloHIQHHnR9r2LwRMVGLuhYwoi0v/lHvDesBAe+y5sWxM6Eky3sFNglFVvP45x8hfD8YvBE6FuZMkHhkJBRPZt2QxsXhaaoza8Do99JwRFujnaw8hhiSgsqmHapTBkUuhSZMgkqBwQT/19TKEgIqUrm4EtK3Y8hvHCL6Fl847XX0BopspmoHoIHP1xGDQ+XN09aDzU7rfPXLSnUBAR6U66DTa+Ec6QWjcfnv4JNK0PZzxl2naev+Ngd6oS3vqpKDTGh+swKmr7uvq9plAQEdlT7S2weSlsXBSGJ74fmqTaW6B9287zJ8rCrVirBsPUi8IeRv1YqB8P9WPC6btFQqEgItKb3MMptB2BsWkxPH0jbFsbOhJMt7LTge9kebSXUQHTPhr2LurHhvCoG9mnV3wrFERE+lI2A1tXwsbFITAe/UYIinQLtG5l5zsHQOgiZEAIjVQFnHp1tLcxDgaM7NWL+BQKIiLFJN0GW5ZFobEkBMdzt28Pjp2OZ1gIikx7uLvesf8Kk86A4Yft1eoVCiIi/Um6LRzP2LR4+97Gc7+A5g3hhknZdhh8AHx61l4tXn0fiYj0J6nyqDvy/bdPO/2a7eOtjX1yeqxCQUSkP+ij019L+7pvERHZgUJBREQ6KRRERKSTQkFERDopFEREpJNCQUREOikURESkk0JBREQ69btuLsxsLbB4L18+FFjXi+X0lmKsSzXlpxhrguKsSzXlrxB1jXP3ht3N1O9C4c0ws5n59P3R14qxLtWUn2KsCYqzLtWUvzjrUvORiIh0UiiIiEinUguFm+IuoAfFWJdqyk8x1gTFWZdqyl9sdZXUMQUREdm1UttTEBGRXVAoiIhIp5IJBTM708zmmdkCM7sqphrGmNmjZjbXzF4xs89E068xs+Vm9kI0nN3HdS0ys5eidc+Mpg02s7+Z2fzo30F9XNNBOdvjBTPbYmaf7ettZWa3mNkaM3s5Z1q328aCH0afsRfN7Mg+rOl/zOzVaL2/M7P6aPp4M2vO2V43FqKmXdTV4/tlZldH22qemZ3RhzXdk1PPIjN7IZreJ9tqF98DsX6uOrn7Pj8ASWAhMBEoB2YDk2OoYwRwZDReB7wGTAauAb4Q4/ZZBAztMu3bwFXR+FXAdTG/f6uAcX29rYCTgSOBl3e3bYCzgb8ABhwHPNOHNb0TSEXj1+XUND53vhi2VbfvV/S5nw1UABOi/5/Jvqipy/PfBb7al9tqF98DsX6uOoZS2VM4Bljg7q+7extwN3BOXxfh7ivd/blofCswFxjV13Xk6Rzg59H4z4H3xljLacBCd9/bK9n3mrs/BmzoMrmnbXMOcLsHTwP1ZjaiL2py97+6ezp6+DQwurfXuzd17cI5wN3u3urubwALCP9P+6wmMzPgPOCu3l7vbmrq6Xsg1s9Vh1IJhVHA0pzHy4j5y9jMxgNTgWeiSVdEu4a39HVTDeDAX81slpldFk3bz91XQvgQA8P6uKZc57Pjf9w4txX0vG2K5XP2UcIvyw4TzOx5M/uHmZ0UQz3dvV/FsK1OAla7+/ycaX26rbp8DxTF56pUQsG6mRbbubhmVgv8Bvisu28BfgLsD0wBVhJ2afvSCe5+JHAW8O9mdnIfr79HZlYOTAd+FU2Ke1vtSuyfMzP7TyAN/DKatBIY6+5Tgc8Bd5rZgD4sqaf3K/ZtBVzAjj82+nRbdfM90OOs3Uwr2LYqlVBYBozJeTwaWBFHIWZWRvgg/NLdfwvg7qvdPePuWeBnFGA3elfcfUX07xrgd9H6V3fsokb/runLmnKcBTzn7qujGmPdVpGetk2snzMz+wjwbuBDHjVGR80z66PxWYS2+wP7qqZdvF9xb6sUcC5wT06tfbatuvseoEg+V6USCjOASWY2IfrleT5wX18XEbVh/h8w192/lzM9t33wfcDLXV9bwJpqzKyuY5xwwPJlwvb5SDTbR4A/9FVNXezway7ObZWjp21zH/Dh6GyR44DNHc0BhWZmZwL/AUx396ac6Q1mlozGJwKTgNf7oqZonT29X/cB55tZhZlNiOp6tq/qAk4HXnX3ZR0T+mpb9fQ9QLF8rgp9pL1YBsIR/NcI6f+fMdVwImG370XghWg4G/gF8FI0/T5gRB/WNJFwFshs4JWObQMMAR4G5kf/Do5he1UD64GBOdP6dFsRAmkl0E74xfaxnrYNYTf/x9Fn7CVgWh/WtIDQ7tzxuboxmvf90fs6G3gOeE8fb6se3y/gP6NtNQ84q69qiqbfBlzeZd4+2Va7+B6I9XPVMaibCxER6VQqzUciIpIHhYKIiHRSKIiISCeFgoiIdFIoiIhIJ4WCiIh0UiiI5MHMpnTp9nm69VIX7Ba6BK/ujWWJvFm6TkEkD2Z2CeGioSsKsOxF0bLX7cFrku6e6e1aRLSnIPuU6EYpc83sZ9ENTP5qZlU9zLu/mT0Q9Q77uJkdHE3/FzN72cxmm9ljUdco1wIfjG6+8kEzu8TMbojmv83MfhLdOOV1Mzsl6hF0rpndlrO+n5jZzKiu/46mfRoYCTxqZo9G0y6wcNOjl83supzXN5rZtWb2DHC8mX3LzOZEPZB+pzBbVEpOIS+X1qChrwfCjVLSwJTo8b3ART3M+zAwKRo/FngkGn8JGBWN10f/XgLckPPazseELhPuJnRHcA6wBXgL4UfXrJxaOrotSAJ/Bw6PHi8iuskRISCWAA1ACngEeG/0nAPndSyL0D2E5dapQcObHbSnIPuiN9z9hWh8FiEodhB1W/xW4FcWbsf4U8IdsQD+CdxmZp8gfIHn44/u7oRAWe3uL3noGfSVnPWfZ2bPAc8DhxLuttXV0cDf3X2th5vm/JJw9zCADKFnTQjB0wLcbGbnAk07LUlkL6TiLkCkAFpzxjNAd81HCWCTu0/p+oS7X25mxwLvAl4ws53m2cU6s13WnwVSUU+gXwCOdveNUbNSZTfL6a7v/A4tHh1HcPe0mR1DuCvd+cAVwNvzqFNkl7SnICXJw01N3jCzf4HOm6MfEY3v7+7PuPtXgXWEvuy3Eu6nu7cGANuAzWa2H+E+ER1yl/0McIqZDY26cb4A+EfXhUV7OgPd/X7gs4Sb2Ii8adpTkFL2IeAnZvYVoIxwXGA28D9mNonwq/3haNoS4Kqoqembe7oid59tZs8TmpNeJzRRdbgJ+IuZrXT3t5nZ1cCj0frvd/fu7mVRB/zBzCqj+a7c05pEuqNTUkVEpJOaj0REpJOaj2SfZ2Y/Bk7oMvl6d781jnpEipmaj0REpJOaj0REpJNCQUREOikURESkk0JBREQ6/X8AhvyEjtcguAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7d1bf7b090>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "xgb3_2 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=205,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=7,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        reg_alpha=2,\n",
    "        reg_lambda=1,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "modelfit(xgb3_2, X_train, y_train, cv_folds = kfold)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "根据测试数据中的均值，选择n_estimators=178"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'colsample_bytree': [0.6, 0.7, 0.8, 0.9],\n",
       " 'subsample': [0.5, 0.6, 0.7, 0.8, 0.9, 1.0]}"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subsample = [i/10.0 for i in range(5,11)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test4_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test4_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.58464, std: 0.00357, params: {'subsample': 0.5, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58231, std: 0.00354, params: {'subsample': 0.6, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58148, std: 0.00347, params: {'subsample': 0.7, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58123, std: 0.00335, params: {'subsample': 0.8, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58097, std: 0.00310, params: {'subsample': 0.9, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58027, std: 0.00340, params: {'subsample': 1.0, 'colsample_bytree': 0.6},\n",
       "  mean: -0.58379, std: 0.00340, params: {'subsample': 0.5, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58270, std: 0.00355, params: {'subsample': 0.6, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58186, std: 0.00371, params: {'subsample': 0.7, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58088, std: 0.00284, params: {'subsample': 0.8, 'colsample_bytree': 0.7},\n",
       "  mean: -0.57998, std: 0.00384, params: {'subsample': 0.9, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58095, std: 0.00328, params: {'subsample': 1.0, 'colsample_bytree': 0.7},\n",
       "  mean: -0.58356, std: 0.00399, params: {'subsample': 0.5, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58191, std: 0.00445, params: {'subsample': 0.6, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58133, std: 0.00371, params: {'subsample': 0.7, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58016, std: 0.00333, params: {'subsample': 0.8, 'colsample_bytree': 0.8},\n",
       "  mean: -0.57997, std: 0.00365, params: {'subsample': 0.9, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58036, std: 0.00366, params: {'subsample': 1.0, 'colsample_bytree': 0.8},\n",
       "  mean: -0.58397, std: 0.00374, params: {'subsample': 0.5, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58220, std: 0.00342, params: {'subsample': 0.6, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58110, std: 0.00328, params: {'subsample': 0.7, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58030, std: 0.00358, params: {'subsample': 0.8, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58062, std: 0.00408, params: {'subsample': 0.9, 'colsample_bytree': 0.9},\n",
       "  mean: -0.58145, std: 0.00343, params: {'subsample': 1.0, 'colsample_bytree': 0.9}],\n",
       " {'colsample_bytree': 0.8, 'subsample': 0.9},\n",
       " -0.5799741664534325)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb4_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=178,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=7,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        reg_alpha=2,\n",
    "        reg_lambda=1,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "gsearch4_1 = GridSearchCV(xgb4_1, param_grid = param_test4_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch4_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch4_1.grid_scores_, gsearch4_1.best_params_,     gsearch4_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb4_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=178,  #数值大没关系，cv会自动返回合适的n_estimators\n",
    "        max_depth=7,\n",
    "        min_child_weight=4,\n",
    "        gamma=0,\n",
    "        reg_alpha=2,\n",
    "        reg_lambda=1,\n",
    "        subsample=0.9,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel=0.8,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "bst = xgb4_1.fit(X_train , y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bathrooms</th>\n",
       "      <th>bedrooms</th>\n",
       "      <th>price</th>\n",
       "      <th>price_bathrooms</th>\n",
       "      <th>price_bedrooms</th>\n",
       "      <th>room_diff</th>\n",
       "      <th>room_num</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Day</th>\n",
       "      <th>...</th>\n",
       "      <th>virtual</th>\n",
       "      <th>walk</th>\n",
       "      <th>walls</th>\n",
       "      <th>war</th>\n",
       "      <th>washer</th>\n",
       "      <th>water</th>\n",
       "      <th>wheelchair</th>\n",
       "      <th>wifi</th>\n",
       "      <th>windows</th>\n",
       "      <th>work</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2950</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>1475.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2850</td>\n",
       "      <td>1425.000000</td>\n",
       "      <td>950.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3758</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>1879.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3300</td>\n",
       "      <td>1650.000000</td>\n",
       "      <td>1100.000000</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4900</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>1633.333333</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2016</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 227 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   bathrooms  bedrooms  price  price_bathrooms  price_bedrooms  room_diff  \\\n",
       "0        1.0         1   2950      1475.000000     1475.000000        0.0   \n",
       "1        1.0         2   2850      1425.000000      950.000000       -1.0   \n",
       "2        1.0         1   3758      1879.000000     1879.000000        0.0   \n",
       "3        1.0         2   3300      1650.000000     1100.000000       -1.0   \n",
       "4        2.0         2   4900      1633.333333     1633.333333        0.0   \n",
       "\n",
       "   room_num  Year  Month  Day  ...   virtual  walk  walls  war  washer  water  \\\n",
       "0       2.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "1       3.0  2016      6   24  ...         0     0      0    1       0      0   \n",
       "2       2.0  2016      6    3  ...         0     0      0    0       0      0   \n",
       "3       3.0  2016      6   11  ...         0     0      0    0       0      0   \n",
       "4       4.0  2016      4   12  ...         0     0      0    1       0      0   \n",
       "\n",
       "   wheelchair  wifi  windows  work  \n",
       "0           0     0        0     0  \n",
       "1           0     0        0     0  \n",
       "2           0     0        0     0  \n",
       "3           1     0        0     0  \n",
       "4           0     0        0     0  \n",
       "\n",
       "[5 rows x 227 columns]"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtest.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/pxt/.local/lib/python2.7/site-packages/sklearn/preprocessing/label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "\n",
    "X_test = np.array(dtest)\n",
    "\n",
    "y_test_pred_xgb = bst.predict(X_test)\n",
    "\n",
    "cvresult.to_csv('my_test.csv',index_label = 'interest_level')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>interest_level</th>\n",
       "      <th>test-mlogloss-mean</th>\n",
       "      <th>test-mlogloss-std</th>\n",
       "      <th>train-mlogloss-mean</th>\n",
       "      <th>train-mlogloss-std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10</td>\n",
       "      <td>0.759061</td>\n",
       "      <td>0.000902</td>\n",
       "      <td>0.752732</td>\n",
       "      <td>0.000994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>0.746131</td>\n",
       "      <td>0.001012</td>\n",
       "      <td>0.739286</td>\n",
       "      <td>0.001108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12</td>\n",
       "      <td>0.734925</td>\n",
       "      <td>0.001164</td>\n",
       "      <td>0.727544</td>\n",
       "      <td>0.001180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td>0.724586</td>\n",
       "      <td>0.001325</td>\n",
       "      <td>0.716813</td>\n",
       "      <td>0.001027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14</td>\n",
       "      <td>0.715464</td>\n",
       "      <td>0.001442</td>\n",
       "      <td>0.707324</td>\n",
       "      <td>0.001069</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   interest_level  test-mlogloss-mean  test-mlogloss-std  train-mlogloss-mean  \\\n",
       "0              10            0.759061           0.000902             0.752732   \n",
       "1              11            0.746131           0.001012             0.739286   \n",
       "2              12            0.734925           0.001164             0.727544   \n",
       "3              13            0.724586           0.001325             0.716813   \n",
       "4              14            0.715464           0.001442             0.707324   \n",
       "\n",
       "   train-mlogloss-std  \n",
       "0            0.000994  \n",
       "1            0.001108  \n",
       "2            0.001180  \n",
       "3            0.001027  \n",
       "4            0.001069  "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = pd.read_csv('my_test.csv')\n",
    "result.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
